1 /* $NetBSD: subr_ipi.c,v 1.7 2019/10/16 18:29:49 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to 34 * invoke functions with a constant argument and synchronous IPIs 35 * with the cross-call support. 36 */ 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.7 2019/10/16 18:29:49 christos Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/types.h> 43 44 #include <sys/atomic.h> 45 #include <sys/evcnt.h> 46 #include <sys/cpu.h> 47 #include <sys/ipi.h> 48 #include <sys/intr.h> 49 #include <sys/kcpuset.h> 50 #include <sys/kmem.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 54 /* 55 * An array of the IPI handlers used for asynchronous invocation. 56 * The lock protects the slot allocation. 57 */ 58 59 typedef struct { 60 ipi_func_t func; 61 void * arg; 62 } ipi_intr_t; 63 64 static kmutex_t ipi_mngmt_lock; 65 static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned; 66 67 /* 68 * Per-CPU mailbox for IPI messages: it is a single cache line storing 69 * up to IPI_MSG_MAX messages. This interface is built on top of the 70 * synchronous IPIs. 71 */ 72 73 #define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *)) 74 #define IPI_MSG_MAX IPI_MSG_SLOTS 75 76 typedef struct { 77 ipi_msg_t * msg[IPI_MSG_SLOTS]; 78 } ipi_mbox_t; 79 80 81 /* Mailboxes for the synchronous IPIs. */ 82 static ipi_mbox_t * ipi_mboxes __read_mostly; 83 static struct evcnt ipi_mboxfull_ev __cacheline_aligned; 84 static void ipi_msg_cpu_handler(void *); 85 86 /* Handler for the synchronous IPIs - it must be zero. */ 87 #define IPI_SYNCH_ID 0 88 89 #ifndef MULTIPROCESSOR 90 #define cpu_ipi(ci) KASSERT(ci == NULL) 91 #endif 92 93 void 94 ipi_sysinit(void) 95 { 96 const size_t len = ncpu * sizeof(ipi_mbox_t); 97 98 /* Initialise the per-CPU bit fields. */ 99 for (u_int i = 0; i < ncpu; i++) { 100 struct cpu_info *ci = cpu_lookup(i); 101 memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend)); 102 } 103 mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE); 104 memset(ipi_intrs, 0, sizeof(ipi_intrs)); 105 106 /* Allocate per-CPU IPI mailboxes. */ 107 ipi_mboxes = kmem_zalloc(len, KM_SLEEP); 108 KASSERT(ipi_mboxes != NULL); 109 110 /* 111 * Register the handler for synchronous IPIs. This mechanism 112 * is built on top of the asynchronous interface. Slot zero is 113 * reserved permanently; it is also handy to use zero as a failure 114 * for other registers (as it is potentially less error-prone). 115 */ 116 ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler; 117 118 evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL, 119 "ipi", "full"); 120 } 121 122 /* 123 * ipi_register: register an asynchronous IPI handler. 124 * 125 * => Returns IPI ID which is greater than zero; on failure - zero. 126 */ 127 u_int 128 ipi_register(ipi_func_t func, void *arg) 129 { 130 mutex_enter(&ipi_mngmt_lock); 131 for (u_int i = 0; i < IPI_MAXREG; i++) { 132 if (ipi_intrs[i].func == NULL) { 133 /* Register the function. */ 134 ipi_intrs[i].func = func; 135 ipi_intrs[i].arg = arg; 136 mutex_exit(&ipi_mngmt_lock); 137 138 KASSERT(i != IPI_SYNCH_ID); 139 return i; 140 } 141 } 142 mutex_exit(&ipi_mngmt_lock); 143 printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n"); 144 return 0; 145 } 146 147 /* 148 * ipi_unregister: release the IPI handler given the ID. 149 */ 150 void 151 ipi_unregister(u_int ipi_id) 152 { 153 ipi_msg_t ipimsg = { .func = __FPTRCAST(ipi_func_t, nullop) }; 154 155 KASSERT(ipi_id != IPI_SYNCH_ID); 156 KASSERT(ipi_id < IPI_MAXREG); 157 158 /* Release the slot. */ 159 mutex_enter(&ipi_mngmt_lock); 160 KASSERT(ipi_intrs[ipi_id].func != NULL); 161 ipi_intrs[ipi_id].func = NULL; 162 163 /* Ensure that there are no IPIs in flight. */ 164 kpreempt_disable(); 165 ipi_broadcast(&ipimsg, false); 166 ipi_wait(&ipimsg); 167 kpreempt_enable(); 168 mutex_exit(&ipi_mngmt_lock); 169 } 170 171 /* 172 * ipi_mark_pending: internal routine to mark an IPI pending on the 173 * specified CPU (which might be curcpu()). 174 */ 175 static bool 176 ipi_mark_pending(u_int ipi_id, struct cpu_info *ci) 177 { 178 const u_int i = ipi_id >> IPI_BITW_SHIFT; 179 const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK); 180 181 KASSERT(ipi_id < IPI_MAXREG); 182 KASSERT(kpreempt_disabled()); 183 184 /* Mark as pending and send an IPI. */ 185 if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) { 186 atomic_or_32(&ci->ci_ipipend[i], bitm); 187 return true; 188 } 189 return false; 190 } 191 192 /* 193 * ipi_trigger: asynchronously send an IPI to the specified CPU. 194 */ 195 void 196 ipi_trigger(u_int ipi_id, struct cpu_info *ci) 197 { 198 199 KASSERT(curcpu() != ci); 200 if (ipi_mark_pending(ipi_id, ci)) { 201 cpu_ipi(ci); 202 } 203 } 204 205 /* 206 * ipi_trigger_multi_internal: the guts of ipi_trigger_multi() and 207 * ipi_trigger_broadcast(). 208 */ 209 static void 210 ipi_trigger_multi_internal(u_int ipi_id, const kcpuset_t *target, 211 bool skip_self) 212 { 213 const cpuid_t selfid = cpu_index(curcpu()); 214 CPU_INFO_ITERATOR cii; 215 struct cpu_info *ci; 216 217 KASSERT(kpreempt_disabled()); 218 KASSERT(target != NULL); 219 220 for (CPU_INFO_FOREACH(cii, ci)) { 221 const cpuid_t cpuid = cpu_index(ci); 222 223 if (!kcpuset_isset(target, cpuid) || cpuid == selfid) { 224 continue; 225 } 226 ipi_trigger(ipi_id, ci); 227 } 228 if (!skip_self && kcpuset_isset(target, selfid)) { 229 ipi_mark_pending(ipi_id, curcpu()); 230 int s = splhigh(); 231 ipi_cpu_handler(); 232 splx(s); 233 } 234 } 235 236 /* 237 * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple 238 * CPUs given the target CPU set. 239 */ 240 void 241 ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target) 242 { 243 ipi_trigger_multi_internal(ipi_id, target, false); 244 } 245 246 /* 247 * ipi_trigger_broadcast: same as ipi_trigger_multi() to kcpuset_attached, 248 * optionally skipping the sending CPU. 249 */ 250 void 251 ipi_trigger_broadcast(u_int ipi_id, bool skip_self) 252 { 253 ipi_trigger_multi_internal(ipi_id, kcpuset_attached, skip_self); 254 } 255 256 /* 257 * put_msg: insert message into the mailbox. 258 */ 259 static inline void 260 put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg) 261 { 262 int count = SPINLOCK_BACKOFF_MIN; 263 again: 264 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 265 if (__predict_true(mbox->msg[i] == NULL) && 266 atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) { 267 return; 268 } 269 } 270 271 /* All slots are full: we have to spin-wait. */ 272 ipi_mboxfull_ev.ev_count++; 273 SPINLOCK_BACKOFF(count); 274 goto again; 275 } 276 277 /* 278 * ipi_cpu_handler: the IPI handler. 279 */ 280 void 281 ipi_cpu_handler(void) 282 { 283 struct cpu_info * const ci = curcpu(); 284 285 /* 286 * Handle asynchronous IPIs: inspect per-CPU bit field, extract 287 * IPI ID numbers and execute functions in those slots. 288 */ 289 for (u_int i = 0; i < IPI_BITWORDS; i++) { 290 uint32_t pending, bit; 291 292 if (ci->ci_ipipend[i] == 0) { 293 continue; 294 } 295 pending = atomic_swap_32(&ci->ci_ipipend[i], 0); 296 #ifndef __HAVE_ATOMIC_AS_MEMBAR 297 membar_producer(); 298 #endif 299 while ((bit = ffs(pending)) != 0) { 300 const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit; 301 ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id]; 302 303 pending &= ~(1U << bit); 304 KASSERT(ipi_hdl->func != NULL); 305 ipi_hdl->func(ipi_hdl->arg); 306 } 307 } 308 } 309 310 /* 311 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox, 312 * execute the passed functions and acknowledge the messages. 313 */ 314 static void 315 ipi_msg_cpu_handler(void *arg __unused) 316 { 317 const struct cpu_info * const ci = curcpu(); 318 ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)]; 319 320 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 321 ipi_msg_t *msg; 322 323 /* Get the message. */ 324 if ((msg = mbox->msg[i]) == NULL) { 325 continue; 326 } 327 mbox->msg[i] = NULL; 328 329 /* Execute the handler. */ 330 KASSERT(msg->func); 331 msg->func(msg->arg); 332 333 /* Ack the request. */ 334 #ifndef __HAVE_ATOMIC_AS_MEMBAR 335 membar_producer(); 336 #endif 337 atomic_dec_uint(&msg->_pending); 338 } 339 } 340 341 /* 342 * ipi_unicast: send an IPI to a single CPU. 343 * 344 * => The CPU must be remote; must not be local. 345 * => The caller must ipi_wait() on the message for completion. 346 */ 347 void 348 ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci) 349 { 350 const cpuid_t id = cpu_index(ci); 351 352 KASSERT(msg->func != NULL); 353 KASSERT(kpreempt_disabled()); 354 KASSERT(curcpu() != ci); 355 356 msg->_pending = 1; 357 membar_producer(); 358 359 put_msg(&ipi_mboxes[id], msg); 360 ipi_trigger(IPI_SYNCH_ID, ci); 361 } 362 363 /* 364 * ipi_multicast: send an IPI to each CPU in the specified set. 365 * 366 * => The caller must ipi_wait() on the message for completion. 367 */ 368 void 369 ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target) 370 { 371 const struct cpu_info * const self = curcpu(); 372 CPU_INFO_ITERATOR cii; 373 struct cpu_info *ci; 374 u_int local; 375 376 KASSERT(msg->func != NULL); 377 KASSERT(kpreempt_disabled()); 378 379 local = !!kcpuset_isset(target, cpu_index(self)); 380 msg->_pending = kcpuset_countset(target) - local; 381 membar_producer(); 382 383 for (CPU_INFO_FOREACH(cii, ci)) { 384 cpuid_t id; 385 386 if (__predict_false(ci == self)) { 387 continue; 388 } 389 id = cpu_index(ci); 390 if (!kcpuset_isset(target, id)) { 391 continue; 392 } 393 put_msg(&ipi_mboxes[id], msg); 394 ipi_trigger(IPI_SYNCH_ID, ci); 395 } 396 if (local) { 397 msg->func(msg->arg); 398 } 399 } 400 401 /* 402 * ipi_broadcast: send an IPI to all CPUs. 403 * 404 * => The caller must ipi_wait() on the message for completion. 405 */ 406 void 407 ipi_broadcast(ipi_msg_t *msg, bool skip_self) 408 { 409 const struct cpu_info * const self = curcpu(); 410 CPU_INFO_ITERATOR cii; 411 struct cpu_info *ci; 412 413 KASSERT(msg->func != NULL); 414 KASSERT(kpreempt_disabled()); 415 416 msg->_pending = ncpu - 1; 417 membar_producer(); 418 419 /* Broadcast IPIs for remote CPUs. */ 420 for (CPU_INFO_FOREACH(cii, ci)) { 421 cpuid_t id; 422 423 if (__predict_false(ci == self)) { 424 continue; 425 } 426 id = cpu_index(ci); 427 put_msg(&ipi_mboxes[id], msg); 428 ipi_trigger(IPI_SYNCH_ID, ci); 429 } 430 431 if (!skip_self) { 432 /* Finally, execute locally. */ 433 msg->func(msg->arg); 434 } 435 } 436 437 /* 438 * ipi_wait: spin-wait until the message is processed. 439 */ 440 void 441 ipi_wait(ipi_msg_t *msg) 442 { 443 int count = SPINLOCK_BACKOFF_MIN; 444 445 while (msg->_pending) { 446 KASSERT(msg->_pending < ncpu); 447 SPINLOCK_BACKOFF(count); 448 } 449 } 450