1 /* $NetBSD: subr_ipi.c,v 1.2 2014/05/25 15:34:19 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to 34 * invoke functions with a constant argument and synchronous IPIs 35 * with the cross-call support. 36 */ 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.2 2014/05/25 15:34:19 rmind Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/types.h> 43 44 #include <sys/atomic.h> 45 #include <sys/evcnt.h> 46 #include <sys/cpu.h> 47 #include <sys/ipi.h> 48 #include <sys/kcpuset.h> 49 #include <sys/kmem.h> 50 #include <sys/lock.h> 51 #include <sys/mutex.h> 52 53 /* 54 * An array of the IPI handlers used for asynchronous invocation. 55 * The lock protects the slot allocation. 56 */ 57 58 typedef struct { 59 ipi_func_t func; 60 void * arg; 61 } ipi_intr_t; 62 63 static kmutex_t ipi_mngmt_lock; 64 static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned; 65 66 /* 67 * Per-CPU mailbox for IPI messages: it is a single cache line storing 68 * up to IPI_MSG_MAX messages. This interface is built on top of the 69 * synchronous IPIs. 70 */ 71 72 #define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *)) 73 #define IPI_MSG_MAX IPI_MSG_SLOTS 74 75 typedef struct { 76 ipi_msg_t * msg[IPI_MSG_SLOTS]; 77 } ipi_mbox_t; 78 79 80 /* Mailboxes for the synchronous IPIs. */ 81 static ipi_mbox_t * ipi_mboxes __read_mostly; 82 static struct evcnt ipi_mboxfull_ev __cacheline_aligned; 83 static void ipi_msg_cpu_handler(void *); 84 85 /* Handler for the synchronous IPIs - it must be zero. */ 86 #define IPI_SYNCH_ID 0 87 88 #ifndef MULTIPROCESSOR 89 #define cpu_ipi(ci) KASSERT(ci == NULL) 90 #endif 91 92 void 93 ipi_sysinit(void) 94 { 95 const size_t len = ncpu * sizeof(ipi_mbox_t); 96 97 /* Initialise the per-CPU bit fields. */ 98 for (u_int i = 0; i < ncpu; i++) { 99 struct cpu_info *ci = cpu_lookup(i); 100 memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend)); 101 } 102 mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE); 103 memset(ipi_intrs, 0, sizeof(ipi_intrs)); 104 105 /* Allocate per-CPU IPI mailboxes. */ 106 ipi_mboxes = kmem_zalloc(len, KM_SLEEP); 107 KASSERT(ipi_mboxes != NULL); 108 109 /* 110 * Register the handler for synchronous IPIs. This mechanism 111 * is built on top of the asynchronous interface. Slot zero is 112 * reserved permanently; it is also handy to use zero as a failure 113 * for other registers (as it is potentially less error-prone). 114 */ 115 ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler; 116 117 evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL, 118 "ipi", "full"); 119 } 120 121 /* 122 * ipi_register: register an asynchronous IPI handler. 123 * 124 * => Returns IPI ID which is greater than zero; on failure - zero. 125 */ 126 u_int 127 ipi_register(ipi_func_t func, void *arg) 128 { 129 mutex_enter(&ipi_mngmt_lock); 130 for (u_int i = 0; i < IPI_MAXREG; i++) { 131 if (ipi_intrs[i].func == NULL) { 132 /* Register the function. */ 133 ipi_intrs[i].func = func; 134 ipi_intrs[i].arg = arg; 135 mutex_exit(&ipi_mngmt_lock); 136 137 KASSERT(i != IPI_SYNCH_ID); 138 return i; 139 } 140 } 141 mutex_exit(&ipi_mngmt_lock); 142 printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n"); 143 return 0; 144 } 145 146 /* 147 * ipi_unregister: release the IPI handler given the ID. 148 */ 149 void 150 ipi_unregister(u_int ipi_id) 151 { 152 ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop }; 153 154 KASSERT(ipi_id != IPI_SYNCH_ID); 155 KASSERT(ipi_id < IPI_MAXREG); 156 157 /* Release the slot. */ 158 mutex_enter(&ipi_mngmt_lock); 159 KASSERT(ipi_intrs[ipi_id].func != NULL); 160 ipi_intrs[ipi_id].func = NULL; 161 162 /* Ensure that there are no IPIs in flight. */ 163 kpreempt_disable(); 164 ipi_broadcast(&ipimsg); 165 ipi_wait(&ipimsg); 166 kpreempt_enable(); 167 mutex_exit(&ipi_mngmt_lock); 168 } 169 170 /* 171 * ipi_trigger: asynchronously send an IPI to the specified CPU. 172 */ 173 void 174 ipi_trigger(u_int ipi_id, struct cpu_info *ci) 175 { 176 const u_int i = ipi_id >> IPI_BITW_SHIFT; 177 const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK); 178 179 KASSERT(ipi_id < IPI_MAXREG); 180 KASSERT(kpreempt_disabled()); 181 KASSERT(curcpu() != ci); 182 183 /* Mark as pending and send an IPI. */ 184 if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) { 185 atomic_or_32(&ci->ci_ipipend[i], bitm); 186 cpu_ipi(ci); 187 } 188 } 189 190 /* 191 * put_msg: insert message into the mailbox. 192 */ 193 static inline void 194 put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg) 195 { 196 int count = SPINLOCK_BACKOFF_MIN; 197 again: 198 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 199 if (__predict_true(mbox->msg[i] == NULL) && 200 atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) { 201 return; 202 } 203 } 204 205 /* All slots are full: we have to spin-wait. */ 206 ipi_mboxfull_ev.ev_count++; 207 SPINLOCK_BACKOFF(count); 208 goto again; 209 } 210 211 /* 212 * ipi_cpu_handler: the IPI handler. 213 */ 214 void 215 ipi_cpu_handler(void) 216 { 217 struct cpu_info * const ci = curcpu(); 218 219 /* 220 * Handle asynchronous IPIs: inspect per-CPU bit field, extract 221 * IPI ID numbers and execute functions in those slots. 222 */ 223 for (u_int i = 0; i < IPI_BITWORDS; i++) { 224 uint32_t pending, bit; 225 226 if (ci->ci_ipipend[i] == 0) { 227 continue; 228 } 229 pending = atomic_swap_32(&ci->ci_ipipend[i], 0); 230 #ifndef __HAVE_ATOMIC_AS_MEMBAR 231 membar_producer(); 232 #endif 233 while ((bit = ffs(pending)) != 0) { 234 const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit; 235 ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id]; 236 237 pending &= ~(1U << bit); 238 KASSERT(ipi_hdl->func != NULL); 239 ipi_hdl->func(ipi_hdl->arg); 240 } 241 } 242 } 243 244 /* 245 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox, 246 * execute the passed functions and acknowledge the messages. 247 */ 248 static void 249 ipi_msg_cpu_handler(void *arg __unused) 250 { 251 const struct cpu_info * const ci = curcpu(); 252 ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)]; 253 254 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 255 ipi_msg_t *msg; 256 257 /* Get the message. */ 258 if ((msg = mbox->msg[i]) == NULL) { 259 continue; 260 } 261 mbox->msg[i] = NULL; 262 263 /* Execute the handler. */ 264 KASSERT(msg->func); 265 msg->func(msg->arg); 266 267 /* Ack the request. */ 268 atomic_dec_uint(&msg->_pending); 269 } 270 } 271 272 /* 273 * ipi_unicast: send an IPI to a single CPU. 274 * 275 * => The CPU must be remote; must not be local. 276 * => The caller must ipi_wait() on the message for completion. 277 */ 278 void 279 ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci) 280 { 281 const cpuid_t id = cpu_index(ci); 282 283 KASSERT(msg->func != NULL); 284 KASSERT(kpreempt_disabled()); 285 KASSERT(curcpu() != ci); 286 287 msg->_pending = 1; 288 membar_producer(); 289 290 put_msg(&ipi_mboxes[id], msg); 291 ipi_trigger(IPI_SYNCH_ID, ci); 292 } 293 294 /* 295 * ipi_multicast: send an IPI to each CPU in the specified set. 296 * 297 * => The caller must ipi_wait() on the message for completion. 298 */ 299 void 300 ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target) 301 { 302 const struct cpu_info * const self = curcpu(); 303 CPU_INFO_ITERATOR cii; 304 struct cpu_info *ci; 305 u_int local; 306 307 KASSERT(msg->func != NULL); 308 KASSERT(kpreempt_disabled()); 309 310 local = !!kcpuset_isset(target, cpu_index(self)); 311 msg->_pending = kcpuset_countset(target) - local; 312 membar_producer(); 313 314 for (CPU_INFO_FOREACH(cii, ci)) { 315 cpuid_t id; 316 317 if (__predict_false(ci == self)) { 318 continue; 319 } 320 id = cpu_index(ci); 321 if (!kcpuset_isset(target, id)) { 322 continue; 323 } 324 put_msg(&ipi_mboxes[id], msg); 325 ipi_trigger(IPI_SYNCH_ID, ci); 326 } 327 if (local) { 328 msg->func(msg->arg); 329 } 330 } 331 332 /* 333 * ipi_broadcast: send an IPI to all CPUs. 334 * 335 * => The caller must ipi_wait() on the message for completion. 336 */ 337 void 338 ipi_broadcast(ipi_msg_t *msg) 339 { 340 const struct cpu_info * const self = curcpu(); 341 CPU_INFO_ITERATOR cii; 342 struct cpu_info *ci; 343 344 KASSERT(msg->func != NULL); 345 KASSERT(kpreempt_disabled()); 346 347 msg->_pending = ncpu - 1; 348 membar_producer(); 349 350 /* Broadcast IPIs for remote CPUs. */ 351 for (CPU_INFO_FOREACH(cii, ci)) { 352 cpuid_t id; 353 354 if (__predict_false(ci == self)) { 355 continue; 356 } 357 id = cpu_index(ci); 358 put_msg(&ipi_mboxes[id], msg); 359 ipi_trigger(IPI_SYNCH_ID, ci); 360 } 361 362 /* Finally, execute locally. */ 363 msg->func(msg->arg); 364 } 365 366 /* 367 * ipi_wait: spin-wait until the message is processed. 368 */ 369 void 370 ipi_wait(ipi_msg_t *msg) 371 { 372 int count = SPINLOCK_BACKOFF_MIN; 373 374 while (msg->_pending) { 375 KASSERT(msg->_pending < ncpu); 376 SPINLOCK_BACKOFF(count); 377 } 378 } 379