1 /* $NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Generic software interrupt framework. 34 * 35 * Overview 36 * 37 * The soft interrupt framework provides a mechanism to schedule a 38 * low priority callback that runs with thread context. It allows 39 * for dynamic registration of software interrupts, and for fair 40 * queueing and prioritization of those interrupts. The callbacks 41 * can be scheduled to run from nearly any point in the kernel: by 42 * code running with thread context, by code running from a 43 * hardware interrupt handler, and at any interrupt priority 44 * level. 45 * 46 * Priority levels 47 * 48 * Since soft interrupt dispatch can be tied to the underlying 49 * architecture's interrupt dispatch code, it can be limited 50 * both by the capabilities of the hardware and the capabilities 51 * of the interrupt dispatch code itself. The number of priority 52 * levels is restricted to four. In order of priority (lowest to 53 * highest) the levels are: clock, bio, net, serial. 54 * 55 * The names are symbolic and in isolation do not have any direct 56 * connection with a particular kind of device activity: they are 57 * only meant as a guide. 58 * 59 * The four priority levels map directly to scheduler priority 60 * levels, and where the architecture implements 'fast' software 61 * interrupts, they also map onto interrupt priorities. The 62 * interrupt priorities are intended to be hidden from machine 63 * independent code, which should use thread-safe mechanisms to 64 * synchronize with software interrupts (for example: mutexes). 65 * 66 * Capabilities 67 * 68 * Software interrupts run with limited machine context. In 69 * particular, they do not posess any address space context. They 70 * should not try to operate on user space addresses, or to use 71 * virtual memory facilities other than those noted as interrupt 72 * safe. 73 * 74 * Unlike hardware interrupts, software interrupts do have thread 75 * context. They may block on synchronization objects, sleep, and 76 * resume execution at a later time. 77 * 78 * Since software interrupts are a limited resource and run with 79 * higher priority than most other LWPs in the system, all 80 * block-and-resume activity by a software interrupt must be kept 81 * short to allow further processing at that level to continue. By 82 * extension, code running with process context must take care to 83 * ensure that any lock that may be taken from a software interrupt 84 * can not be held for more than a short period of time. 85 * 86 * The kernel does not allow software interrupts to use facilities 87 * or perform actions that may block for a significant amount of 88 * time. This means that it's not valid for a software interrupt 89 * to sleep on condition variables or wait for resources to become 90 * available (for example, memory). 91 * 92 * Per-CPU operation 93 * 94 * If a soft interrupt is triggered on a CPU, it can only be 95 * dispatched on the same CPU. Each LWP dedicated to handling a 96 * soft interrupt is bound to its home CPU, so if the LWP blocks 97 * and needs to run again, it can only run there. Nearly all data 98 * structures used to manage software interrupts are per-CPU. 99 * 100 * The per-CPU requirement is intended to reduce "ping-pong" of 101 * cache lines between CPUs: lines occupied by data structures 102 * used to manage the soft interrupts, and lines occupied by data 103 * items being passed down to the soft interrupt. As a positive 104 * side effect, this also means that the soft interrupt dispatch 105 * code does not need to to use spinlocks to synchronize. 106 * 107 * Generic implementation 108 * 109 * A generic, low performance implementation is provided that 110 * works across all architectures, with no machine-dependent 111 * modifications needed. This implementation uses the scheduler, 112 * and so has a number of restrictions: 113 * 114 * 1) The software interrupts are not currently preemptive, so 115 * must wait for the currently executing LWP to yield the CPU. 116 * This can introduce latency. 117 * 118 * 2) An expensive context switch is required for a software 119 * interrupt to be handled. 120 * 121 * 'Fast' software interrupts 122 * 123 * If an architectures defines __HAVE_FAST_SOFTINTS, it implements 124 * the fast mechanism. Threads running either in the kernel or in 125 * userspace will be interrupted, but will not be preempted. When 126 * the soft interrupt completes execution, the interrupted LWP 127 * is resumed. Interrupt dispatch code must provide the minimum 128 * level of context necessary for the soft interrupt to block and 129 * be resumed at a later time. The machine-dependent dispatch 130 * path looks something like the following: 131 * 132 * softintr() 133 * { 134 * go to IPL_HIGH if necessary for switch; 135 * save any necessary registers in a format that can be 136 * restored by cpu_switchto if the softint blocks; 137 * arrange for cpu_switchto() to restore into the 138 * trampoline function; 139 * identify LWP to handle this interrupt; 140 * switch to the LWP's stack; 141 * switch register stacks, if necessary; 142 * assign new value of curlwp; 143 * call MI softint_dispatch, passing old curlwp and IPL 144 * to execute interrupt at; 145 * switch back to old stack; 146 * switch back to old register stack, if necessary; 147 * restore curlwp; 148 * return to interrupted LWP; 149 * } 150 * 151 * If the soft interrupt blocks, a trampoline function is returned 152 * to in the context of the interrupted LWP, as arranged for by 153 * softint(): 154 * 155 * softint_ret() 156 * { 157 * unlock soft interrupt LWP; 158 * resume interrupt processing, likely returning to 159 * interrupted LWP or dispatching another, different 160 * interrupt; 161 * } 162 * 163 * Once the soft interrupt has fired (and even if it has blocked), 164 * no further soft interrupts at that level will be triggered by 165 * MI code until the soft interrupt handler has ceased execution. 166 * If a soft interrupt handler blocks and is resumed, it resumes 167 * execution as a normal LWP (kthread) and gains VM context. Only 168 * when it has completed and is ready to fire again will it 169 * interrupt other threads. 170 */ 171 172 #include <sys/cdefs.h> 173 __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.76 2024/03/01 04:32:38 mrg Exp $"); 174 175 #include <sys/param.h> 176 #include <sys/proc.h> 177 #include <sys/intr.h> 178 #include <sys/ipi.h> 179 #include <sys/lock.h> 180 #include <sys/mutex.h> 181 #include <sys/kernel.h> 182 #include <sys/kthread.h> 183 #include <sys/evcnt.h> 184 #include <sys/cpu.h> 185 #include <sys/xcall.h> 186 #include <sys/psref.h> 187 #include <sys/sdt.h> 188 189 #include <uvm/uvm_extern.h> 190 191 /* This could overlap with signal info in struct lwp. */ 192 typedef struct softint { 193 SIMPLEQ_HEAD(, softhand) si_q; 194 struct lwp *si_lwp; 195 struct cpu_info *si_cpu; 196 uintptr_t si_machdep; 197 struct evcnt si_evcnt; 198 struct evcnt si_evcnt_block; 199 volatile int si_active; 200 int si_ipl; 201 char si_name[8]; 202 char si_name_block[8+6]; 203 } softint_t; 204 205 typedef struct softhand { 206 SIMPLEQ_ENTRY(softhand) sh_q; 207 void (*sh_func)(void *); 208 void *sh_arg; 209 softint_t *sh_isr; 210 u_int sh_flags; 211 u_int sh_ipi_id; 212 } softhand_t; 213 214 typedef struct softcpu { 215 struct cpu_info *sc_cpu; 216 softint_t sc_int[SOFTINT_COUNT]; 217 softhand_t sc_hand[1]; 218 } softcpu_t; 219 220 static void softint_thread(void *); 221 222 u_int softint_bytes = 32768; 223 u_int softint_timing; 224 static u_int softint_max; 225 static kmutex_t softint_lock; 226 227 SDT_PROBE_DEFINE4(sdt, kernel, softint, establish, 228 "void *"/*sih*/, 229 "void (*)(void *)"/*func*/, 230 "void *"/*arg*/, 231 "unsigned"/*flags*/); 232 233 SDT_PROBE_DEFINE1(sdt, kernel, softint, disestablish, 234 "void *"/*sih*/); 235 236 SDT_PROBE_DEFINE2(sdt, kernel, softint, schedule, 237 "void *"/*sih*/, 238 "struct cpu_info *"/*ci*/); 239 240 SDT_PROBE_DEFINE4(sdt, kernel, softint, entry, 241 "void *"/*sih*/, 242 "void (*)(void *)"/*func*/, 243 "void *"/*arg*/, 244 "unsigned"/*flags*/); 245 246 SDT_PROBE_DEFINE4(sdt, kernel, softint, return, 247 "void *"/*sih*/, 248 "void (*)(void *)"/*func*/, 249 "void *"/*arg*/, 250 "unsigned"/*flags*/); 251 252 /* 253 * softint_init_isr: 254 * 255 * Initialize a single interrupt level for a single CPU. 256 */ 257 static void 258 softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level, 259 int ipl) 260 { 261 struct cpu_info *ci; 262 softint_t *si; 263 int error; 264 265 si = &sc->sc_int[level]; 266 ci = sc->sc_cpu; 267 si->si_cpu = ci; 268 269 SIMPLEQ_INIT(&si->si_q); 270 271 error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR | 272 KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp, 273 "soft%s/%u", desc, ci->ci_index); 274 if (error != 0) 275 panic("softint_init_isr: error %d", error); 276 277 snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc, 278 ci->ci_index); 279 evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL, 280 "softint", si->si_name); 281 snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u", 282 desc, ci->ci_index); 283 evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL, 284 "softint", si->si_name_block); 285 286 si->si_ipl = ipl; 287 si->si_lwp->l_private = si; 288 softint_init_md(si->si_lwp, level, &si->si_machdep); 289 } 290 291 /* 292 * softint_init: 293 * 294 * Initialize per-CPU data structures. Called from mi_cpu_attach(). 295 */ 296 void 297 softint_init(struct cpu_info *ci) 298 { 299 static struct cpu_info *first; 300 softcpu_t *sc, *scfirst; 301 softhand_t *sh, *shmax; 302 303 if (first == NULL) { 304 /* Boot CPU. */ 305 first = ci; 306 mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE); 307 softint_bytes = round_page(softint_bytes); 308 softint_max = (softint_bytes - sizeof(softcpu_t)) / 309 sizeof(softhand_t); 310 } 311 312 /* Use uvm_km(9) for persistent, page-aligned allocation. */ 313 sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0, 314 UVM_KMF_WIRED | UVM_KMF_ZERO); 315 if (sc == NULL) 316 panic("softint_init_cpu: cannot allocate memory"); 317 318 ci->ci_data.cpu_softcpu = sc; 319 ci->ci_data.cpu_softints = 0; 320 sc->sc_cpu = ci; 321 322 softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET, 323 IPL_SOFTNET); 324 softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO, 325 IPL_SOFTBIO); 326 softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK, 327 IPL_SOFTCLOCK); 328 softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL, 329 IPL_SOFTSERIAL); 330 331 if (first != ci) { 332 mutex_enter(&softint_lock); 333 scfirst = first->ci_data.cpu_softcpu; 334 sh = sc->sc_hand; 335 memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max); 336 /* Update pointers for this CPU. */ 337 for (shmax = sh + softint_max; sh < shmax; sh++) { 338 if (sh->sh_func == NULL) 339 continue; 340 sh->sh_isr = 341 &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK]; 342 } 343 mutex_exit(&softint_lock); 344 } 345 } 346 347 /* 348 * softint_establish: 349 * 350 * Register a software interrupt handler. 351 */ 352 void * 353 softint_establish(u_int flags, void (*func)(void *), void *arg) 354 { 355 CPU_INFO_ITERATOR cii; 356 struct cpu_info *ci; 357 softcpu_t *sc; 358 softhand_t *sh; 359 u_int level, index; 360 u_int ipi_id = 0; 361 void *sih; 362 363 level = (flags & SOFTINT_LVLMASK); 364 KASSERT(level < SOFTINT_COUNT); 365 KASSERT((flags & SOFTINT_IMPMASK) == 0); 366 367 mutex_enter(&softint_lock); 368 369 /* Find a free slot. */ 370 sc = curcpu()->ci_data.cpu_softcpu; 371 for (index = 1; index < softint_max; index++) { 372 if (sc->sc_hand[index].sh_func == NULL) 373 break; 374 } 375 if (index == softint_max) { 376 mutex_exit(&softint_lock); 377 printf("WARNING: softint_establish: table full, " 378 "increase softint_bytes\n"); 379 return NULL; 380 } 381 sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc); 382 383 if (flags & SOFTINT_RCPU) { 384 if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) { 385 mutex_exit(&softint_lock); 386 return NULL; 387 } 388 } 389 390 /* Set up the handler on each CPU. */ 391 if (ncpu < 2) { 392 /* XXX hack for machines with no CPU_INFO_FOREACH() early on */ 393 sc = curcpu()->ci_data.cpu_softcpu; 394 sh = &sc->sc_hand[index]; 395 sh->sh_isr = &sc->sc_int[level]; 396 sh->sh_func = func; 397 sh->sh_arg = arg; 398 sh->sh_flags = flags; 399 sh->sh_ipi_id = ipi_id; 400 } else for (CPU_INFO_FOREACH(cii, ci)) { 401 sc = ci->ci_data.cpu_softcpu; 402 sh = &sc->sc_hand[index]; 403 sh->sh_isr = &sc->sc_int[level]; 404 sh->sh_func = func; 405 sh->sh_arg = arg; 406 sh->sh_flags = flags; 407 sh->sh_ipi_id = ipi_id; 408 } 409 mutex_exit(&softint_lock); 410 411 SDT_PROBE4(sdt, kernel, softint, establish, sih, func, arg, flags); 412 413 return sih; 414 } 415 416 /* 417 * softint_disestablish: 418 * 419 * Unregister a software interrupt handler. The soft interrupt could 420 * still be active at this point, but the caller commits not to try 421 * and trigger it again once this call is made. The caller must not 422 * hold any locks that could be taken from soft interrupt context, 423 * because we will wait for the softint to complete if it's still 424 * running. 425 */ 426 void 427 softint_disestablish(void *arg) 428 { 429 CPU_INFO_ITERATOR cii; 430 struct cpu_info *ci; 431 softcpu_t *sc; 432 softhand_t *sh; 433 uintptr_t offset; 434 435 offset = (uintptr_t)arg; 436 KASSERT(offset != 0); 437 KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u", 438 offset, softint_bytes); 439 440 /* 441 * Unregister IPI handler if there is any. Note: there is no need 442 * to disable preemption here - ID is stable. 443 */ 444 sc = curcpu()->ci_data.cpu_softcpu; 445 sh = (softhand_t *)((uint8_t *)sc + offset); 446 if (sh->sh_ipi_id) { 447 ipi_unregister(sh->sh_ipi_id); 448 } 449 450 /* 451 * Run a dummy softint at the same level on all CPUs and wait for 452 * completion, to make sure this softint is no longer running 453 * anywhere. 454 */ 455 xc_barrier(XC_HIGHPRI_IPL(sh->sh_isr->si_ipl)); 456 457 /* 458 * Notify dtrace probe when the old softint can't be running 459 * any more, but before it can be recycled for a new softint. 460 */ 461 SDT_PROBE1(sdt, kernel, softint, disestablish, arg); 462 463 /* Clear the handler on each CPU. */ 464 mutex_enter(&softint_lock); 465 for (CPU_INFO_FOREACH(cii, ci)) { 466 sc = ci->ci_data.cpu_softcpu; 467 sh = (softhand_t *)((uint8_t *)sc + offset); 468 KASSERT(sh->sh_func != NULL); 469 sh->sh_func = NULL; 470 } 471 mutex_exit(&softint_lock); 472 } 473 474 /* 475 * softint_schedule: 476 * 477 * Trigger a software interrupt. Must be called from a hardware 478 * interrupt handler, or with preemption disabled (since we are 479 * using the value of curcpu()). 480 */ 481 void 482 softint_schedule(void *arg) 483 { 484 softhand_t *sh; 485 softint_t *si; 486 uintptr_t offset; 487 int s; 488 489 SDT_PROBE2(sdt, kernel, softint, schedule, arg, /*ci*/NULL); 490 491 /* 492 * If this assert fires, rather than disabling preemption explicitly 493 * to make it stop, consider that you are probably using a softint 494 * when you don't need to. 495 */ 496 KASSERT(kpreempt_disabled()); 497 498 /* Find the handler record for this CPU. */ 499 offset = (uintptr_t)arg; 500 KASSERT(offset != 0); 501 KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u", 502 offset, softint_bytes); 503 sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset); 504 505 /* If it's already pending there's nothing to do. */ 506 if ((sh->sh_flags & SOFTINT_PENDING) != 0) { 507 return; 508 } 509 510 /* 511 * Enqueue the handler into the LWP's pending list. 512 * If the LWP is completely idle, then make it run. 513 */ 514 s = splhigh(); 515 if ((sh->sh_flags & SOFTINT_PENDING) == 0) { 516 si = sh->sh_isr; 517 sh->sh_flags |= SOFTINT_PENDING; 518 SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q); 519 if (si->si_active == 0) { 520 si->si_active = 1; 521 softint_trigger(si->si_machdep); 522 } 523 } 524 splx(s); 525 } 526 527 /* 528 * softint_schedule_cpu: 529 * 530 * Trigger a software interrupt on a target CPU. This invokes 531 * softint_schedule() for the local CPU or send an IPI to invoke 532 * this routine on the remote CPU. Preemption must be disabled. 533 */ 534 void 535 softint_schedule_cpu(void *arg, struct cpu_info *ci) 536 { 537 KASSERT(kpreempt_disabled()); 538 539 if (curcpu() != ci) { 540 const softcpu_t *sc = ci->ci_data.cpu_softcpu; 541 const uintptr_t offset = (uintptr_t)arg; 542 const softhand_t *sh; 543 544 SDT_PROBE2(sdt, kernel, softint, schedule, arg, ci); 545 sh = (const softhand_t *)((const uint8_t *)sc + offset); 546 KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0); 547 ipi_trigger(sh->sh_ipi_id, ci); 548 return; 549 } 550 551 /* Just a local CPU. */ 552 softint_schedule(arg); 553 } 554 555 /* 556 * softint_execute: 557 * 558 * Invoke handlers for the specified soft interrupt. 559 * Must be entered at splhigh. Will drop the priority 560 * to the level specified, but returns back at splhigh. 561 */ 562 static inline void 563 softint_execute(lwp_t *l, int s) 564 { 565 softint_t *si = l->l_private; 566 softhand_t *sh; 567 568 KASSERT(si->si_lwp == curlwp); 569 KASSERT(si->si_cpu == curcpu()); 570 KASSERT(si->si_lwp->l_wchan == NULL); 571 KASSERT(si->si_active); 572 KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d", 573 l, l->l_nopreempt); 574 575 /* 576 * Note: due to priority inheritance we may have interrupted a 577 * higher priority LWP. Since the soft interrupt must be quick 578 * and is non-preemptable, we don't bother yielding. 579 */ 580 581 while (!SIMPLEQ_EMPTY(&si->si_q)) { 582 /* 583 * Pick the longest waiting handler to run. We block 584 * interrupts but do not lock in order to do this, as 585 * we are protecting against the local CPU only. 586 */ 587 sh = SIMPLEQ_FIRST(&si->si_q); 588 SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q); 589 KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0); 590 sh->sh_flags ^= SOFTINT_PENDING; 591 splx(s); 592 593 /* Run the handler. */ 594 SDT_PROBE4(sdt, kernel, softint, entry, 595 ((const char *)sh - 596 (const char *)curcpu()->ci_data.cpu_softcpu), 597 sh->sh_func, sh->sh_arg, sh->sh_flags); 598 if (__predict_true((sh->sh_flags & SOFTINT_MPSAFE) != 0)) { 599 (*sh->sh_func)(sh->sh_arg); 600 } else { 601 KERNEL_LOCK(1, l); 602 (*sh->sh_func)(sh->sh_arg); 603 KERNEL_UNLOCK_ONE(l); 604 } 605 SDT_PROBE4(sdt, kernel, softint, return, 606 ((const char *)sh - 607 (const char *)curcpu()->ci_data.cpu_softcpu), 608 sh->sh_func, sh->sh_arg, sh->sh_flags); 609 610 /* Diagnostic: check that spin-locks have not leaked. */ 611 KASSERTMSG(curcpu()->ci_mtx_count == 0, 612 "%s: ci_mtx_count (%d) != 0, sh_func %p\n", 613 __func__, curcpu()->ci_mtx_count, sh->sh_func); 614 /* Diagnostic: check that psrefs have not leaked. */ 615 KASSERTMSG(l->l_psrefs == 0, "%s: l_psrefs=%d, sh_func=%p\n", 616 __func__, l->l_psrefs, sh->sh_func); 617 /* Diagnostic: check that biglocks have not leaked. */ 618 KASSERTMSG(l->l_blcnt == 0, 619 "%s: sh_func=%p leaked %d biglocks", 620 __func__, sh->sh_func, curlwp->l_blcnt); 621 /* Diagnostic: check that LWP nopreempt remains zero. */ 622 KASSERTMSG(l->l_nopreempt == 0, 623 "%s: lwp %p nopreempt %d func %p", 624 __func__, l, l->l_nopreempt, sh->sh_func); 625 626 (void)splhigh(); 627 } 628 629 PSREF_DEBUG_BARRIER(); 630 631 CPU_COUNT(CPU_COUNT_NSOFT, 1); 632 633 KASSERT(si->si_cpu == curcpu()); 634 KASSERT(si->si_lwp->l_wchan == NULL); 635 KASSERT(si->si_active); 636 si->si_evcnt.ev_count++; 637 si->si_active = 0; 638 } 639 640 /* 641 * softint_block: 642 * 643 * Update statistics when the soft interrupt blocks. 644 */ 645 void 646 softint_block(lwp_t *l) 647 { 648 softint_t *si = l->l_private; 649 650 KASSERT((l->l_pflag & LP_INTR) != 0); 651 si->si_evcnt_block.ev_count++; 652 } 653 654 #ifndef __HAVE_FAST_SOFTINTS 655 656 #ifdef __HAVE_PREEMPTION 657 #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS 658 #endif 659 660 /* 661 * softint_init_md: 662 * 663 * Slow path: perform machine-dependent initialization. 664 */ 665 void 666 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) 667 { 668 struct proc *p; 669 softint_t *si; 670 671 *machdep = (1 << level); 672 si = l->l_private; 673 p = l->l_proc; 674 675 mutex_enter(p->p_lock); 676 lwp_lock(l); 677 /* Cheat and make the KASSERT in softint_thread() happy. */ 678 si->si_active = 1; 679 setrunnable(l); 680 /* LWP now unlocked */ 681 mutex_exit(p->p_lock); 682 } 683 684 /* 685 * softint_trigger: 686 * 687 * Slow path: cause a soft interrupt handler to begin executing. 688 * Called at IPL_HIGH. 689 */ 690 void 691 softint_trigger(uintptr_t machdep) 692 { 693 struct cpu_info *ci; 694 lwp_t *l; 695 696 ci = curcpu(); 697 ci->ci_data.cpu_softints |= machdep; 698 l = ci->ci_onproc; 699 700 /* 701 * Arrange for mi_switch() to be called. If called from interrupt 702 * mode, we don't know if curlwp is executing in kernel or user, so 703 * post an AST and have it take a trip through userret(). If not in 704 * interrupt mode, curlwp is running in kernel and will notice the 705 * resched soon enough; avoid the AST. 706 */ 707 if (l == ci->ci_data.cpu_idlelwp) { 708 atomic_or_uint(&ci->ci_want_resched, 709 RESCHED_IDLE | RESCHED_UPREEMPT); 710 } else { 711 atomic_or_uint(&ci->ci_want_resched, RESCHED_UPREEMPT); 712 if (cpu_intr_p()) { 713 cpu_signotify(l); 714 } 715 } 716 } 717 718 /* 719 * softint_thread: 720 * 721 * Slow path: MI software interrupt dispatch. 722 */ 723 void 724 softint_thread(void *cookie) 725 { 726 softint_t *si; 727 lwp_t *l; 728 int s; 729 730 l = curlwp; 731 si = l->l_private; 732 733 for (;;) { 734 /* Clear pending status and run it. */ 735 s = splhigh(); 736 l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep; 737 softint_execute(l, s); 738 splx(s); 739 740 /* Interrupts allowed to run again before switching. */ 741 lwp_lock(l); 742 l->l_stat = LSIDL; 743 spc_lock(l->l_cpu); 744 mi_switch(l); 745 } 746 } 747 748 /* 749 * softint_picklwp: 750 * 751 * Slow path: called from mi_switch() to pick the highest priority 752 * soft interrupt LWP that needs to run. 753 */ 754 lwp_t * 755 softint_picklwp(void) 756 { 757 struct cpu_info *ci; 758 u_int mask; 759 softint_t *si; 760 lwp_t *l; 761 762 ci = curcpu(); 763 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 764 mask = ci->ci_data.cpu_softints; 765 766 if ((mask & (1 << SOFTINT_SERIAL)) != 0) { 767 l = si[SOFTINT_SERIAL].si_lwp; 768 } else if ((mask & (1 << SOFTINT_NET)) != 0) { 769 l = si[SOFTINT_NET].si_lwp; 770 } else if ((mask & (1 << SOFTINT_BIO)) != 0) { 771 l = si[SOFTINT_BIO].si_lwp; 772 } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) { 773 l = si[SOFTINT_CLOCK].si_lwp; 774 } else { 775 panic("softint_picklwp"); 776 } 777 778 return l; 779 } 780 781 #else /* !__HAVE_FAST_SOFTINTS */ 782 783 /* 784 * softint_thread: 785 * 786 * Fast path: the LWP is switched to without restoring any state, 787 * so we should not arrive here - there is a direct handoff between 788 * the interrupt stub and softint_dispatch(). 789 */ 790 void 791 softint_thread(void *cookie) 792 { 793 794 panic("softint_thread"); 795 } 796 797 /* 798 * softint_dispatch: 799 * 800 * Fast path: entry point from machine-dependent code. 801 */ 802 void 803 softint_dispatch(lwp_t *pinned, int s) 804 { 805 struct bintime now; 806 u_int timing; 807 lwp_t *l; 808 809 #ifdef DIAGNOSTIC 810 if ((pinned->l_pflag & LP_RUNNING) == 0 || curlwp->l_stat != LSIDL) { 811 struct lwp *onproc = curcpu()->ci_onproc; 812 int s2 = splhigh(); 813 printf("curcpu=%d, spl=%d curspl=%d\n" 814 "onproc=%p => l_stat=%d l_flag=%08x l_cpu=%d\n" 815 "curlwp=%p => l_stat=%d l_flag=%08x l_cpu=%d\n" 816 "pinned=%p => l_stat=%d l_flag=%08x l_cpu=%d\n", 817 cpu_index(curcpu()), s, s2, onproc, onproc->l_stat, 818 onproc->l_flag, cpu_index(onproc->l_cpu), curlwp, 819 curlwp->l_stat, curlwp->l_flag, 820 cpu_index(curlwp->l_cpu), pinned, pinned->l_stat, 821 pinned->l_flag, cpu_index(pinned->l_cpu)); 822 splx(s2); 823 panic("softint screwup"); 824 } 825 #endif 826 827 /* 828 * Note the interrupted LWP, and mark the current LWP as running 829 * before proceeding. Although this must as a rule be done with 830 * the LWP locked, at this point no external agents will want to 831 * modify the interrupt LWP's state. 832 */ 833 timing = softint_timing; 834 l = curlwp; 835 l->l_switchto = pinned; 836 l->l_stat = LSONPROC; 837 838 /* 839 * Dispatch the interrupt. If softints are being timed, charge 840 * for it. 841 */ 842 if (timing) { 843 binuptime(&l->l_stime); 844 membar_producer(); /* for calcru */ 845 l->l_pflag |= LP_TIMEINTR; 846 } 847 l->l_pflag |= LP_RUNNING; 848 softint_execute(l, s); 849 if (timing) { 850 binuptime(&now); 851 updatertime(l, &now); 852 l->l_pflag &= ~LP_TIMEINTR; 853 } 854 855 /* 856 * If we blocked while handling the interrupt, the pinned LWP is 857 * gone and we are now running as a kthread, so find another LWP to 858 * run. softint_dispatch() won't be reentered until the priority is 859 * finally dropped to IPL_NONE on entry to the next LWP on this CPU. 860 */ 861 l->l_stat = LSIDL; 862 if (l->l_switchto == NULL) { 863 lwp_lock(l); 864 spc_lock(l->l_cpu); 865 mi_switch(l); 866 /* NOTREACHED */ 867 } 868 l->l_switchto = NULL; 869 l->l_pflag &= ~LP_RUNNING; 870 } 871 872 #endif /* !__HAVE_FAST_SOFTINTS */ 873