1 /* $NetBSD: kern_softint.c,v 1.45 2017/12/28 03:39:48 msaitoh Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Generic software interrupt framework. 34 * 35 * Overview 36 * 37 * The soft interrupt framework provides a mechanism to schedule a 38 * low priority callback that runs with thread context. It allows 39 * for dynamic registration of software interrupts, and for fair 40 * queueing and prioritization of those interrupts. The callbacks 41 * can be scheduled to run from nearly any point in the kernel: by 42 * code running with thread context, by code running from a 43 * hardware interrupt handler, and at any interrupt priority 44 * level. 45 * 46 * Priority levels 47 * 48 * Since soft interrupt dispatch can be tied to the underlying 49 * architecture's interrupt dispatch code, it can be limited 50 * both by the capabilities of the hardware and the capabilities 51 * of the interrupt dispatch code itself. The number of priority 52 * levels is restricted to four. In order of priority (lowest to 53 * highest) the levels are: clock, bio, net, serial. 54 * 55 * The names are symbolic and in isolation do not have any direct 56 * connection with a particular kind of device activity: they are 57 * only meant as a guide. 58 * 59 * The four priority levels map directly to scheduler priority 60 * levels, and where the architecture implements 'fast' software 61 * interrupts, they also map onto interrupt priorities. The 62 * interrupt priorities are intended to be hidden from machine 63 * independent code, which should use thread-safe mechanisms to 64 * synchronize with software interrupts (for example: mutexes). 65 * 66 * Capabilities 67 * 68 * Software interrupts run with limited machine context. In 69 * particular, they do not posess any address space context. They 70 * should not try to operate on user space addresses, or to use 71 * virtual memory facilities other than those noted as interrupt 72 * safe. 73 * 74 * Unlike hardware interrupts, software interrupts do have thread 75 * context. They may block on synchronization objects, sleep, and 76 * resume execution at a later time. 77 * 78 * Since software interrupts are a limited resource and run with 79 * higher priority than most other LWPs in the system, all 80 * block-and-resume activity by a software interrupt must be kept 81 * short to allow futher processing at that level to continue. By 82 * extension, code running with process context must take care to 83 * ensure that any lock that may be taken from a software interrupt 84 * can not be held for more than a short period of time. 85 * 86 * The kernel does not allow software interrupts to use facilities 87 * or perform actions that may block for a significant amount of 88 * time. This means that it's not valid for a software interrupt 89 * to sleep on condition variables or wait for resources to become 90 * available (for example, memory). 91 * 92 * Per-CPU operation 93 * 94 * If a soft interrupt is triggered on a CPU, it can only be 95 * dispatched on the same CPU. Each LWP dedicated to handling a 96 * soft interrupt is bound to its home CPU, so if the LWP blocks 97 * and needs to run again, it can only run there. Nearly all data 98 * structures used to manage software interrupts are per-CPU. 99 * 100 * The per-CPU requirement is intended to reduce "ping-pong" of 101 * cache lines between CPUs: lines occupied by data structures 102 * used to manage the soft interrupts, and lines occupied by data 103 * items being passed down to the soft interrupt. As a positive 104 * side effect, this also means that the soft interrupt dispatch 105 * code does not need to to use spinlocks to synchronize. 106 * 107 * Generic implementation 108 * 109 * A generic, low performance implementation is provided that 110 * works across all architectures, with no machine-dependent 111 * modifications needed. This implementation uses the scheduler, 112 * and so has a number of restrictions: 113 * 114 * 1) The software interrupts are not currently preemptive, so 115 * must wait for the currently executing LWP to yield the CPU. 116 * This can introduce latency. 117 * 118 * 2) An expensive context switch is required for a software 119 * interrupt to be handled. 120 * 121 * 'Fast' software interrupts 122 * 123 * If an architectures defines __HAVE_FAST_SOFTINTS, it implements 124 * the fast mechanism. Threads running either in the kernel or in 125 * userspace will be interrupted, but will not be preempted. When 126 * the soft interrupt completes execution, the interrupted LWP 127 * is resumed. Interrupt dispatch code must provide the minimum 128 * level of context necessary for the soft interrupt to block and 129 * be resumed at a later time. The machine-dependent dispatch 130 * path looks something like the following: 131 * 132 * softintr() 133 * { 134 * go to IPL_HIGH if necessary for switch; 135 * save any necessary registers in a format that can be 136 * restored by cpu_switchto if the softint blocks; 137 * arrange for cpu_switchto() to restore into the 138 * trampoline function; 139 * identify LWP to handle this interrupt; 140 * switch to the LWP's stack; 141 * switch register stacks, if necessary; 142 * assign new value of curlwp; 143 * call MI softint_dispatch, passing old curlwp and IPL 144 * to execute interrupt at; 145 * switch back to old stack; 146 * switch back to old register stack, if necessary; 147 * restore curlwp; 148 * return to interrupted LWP; 149 * } 150 * 151 * If the soft interrupt blocks, a trampoline function is returned 152 * to in the context of the interrupted LWP, as arranged for by 153 * softint(): 154 * 155 * softint_ret() 156 * { 157 * unlock soft interrupt LWP; 158 * resume interrupt processing, likely returning to 159 * interrupted LWP or dispatching another, different 160 * interrupt; 161 * } 162 * 163 * Once the soft interrupt has fired (and even if it has blocked), 164 * no further soft interrupts at that level will be triggered by 165 * MI code until the soft interrupt handler has ceased execution. 166 * If a soft interrupt handler blocks and is resumed, it resumes 167 * execution as a normal LWP (kthread) and gains VM context. Only 168 * when it has completed and is ready to fire again will it 169 * interrupt other threads. 170 */ 171 172 #include <sys/cdefs.h> 173 __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.45 2017/12/28 03:39:48 msaitoh Exp $"); 174 175 #include <sys/param.h> 176 #include <sys/proc.h> 177 #include <sys/intr.h> 178 #include <sys/ipi.h> 179 #include <sys/mutex.h> 180 #include <sys/kernel.h> 181 #include <sys/kthread.h> 182 #include <sys/evcnt.h> 183 #include <sys/cpu.h> 184 #include <sys/xcall.h> 185 #include <sys/pserialize.h> 186 187 #include <net/netisr.h> 188 189 #include <uvm/uvm_extern.h> 190 191 /* This could overlap with signal info in struct lwp. */ 192 typedef struct softint { 193 SIMPLEQ_HEAD(, softhand) si_q; 194 struct lwp *si_lwp; 195 struct cpu_info *si_cpu; 196 uintptr_t si_machdep; 197 struct evcnt si_evcnt; 198 struct evcnt si_evcnt_block; 199 int si_active; 200 char si_name[8]; 201 char si_name_block[8+6]; 202 } softint_t; 203 204 typedef struct softhand { 205 SIMPLEQ_ENTRY(softhand) sh_q; 206 void (*sh_func)(void *); 207 void *sh_arg; 208 softint_t *sh_isr; 209 u_int sh_flags; 210 u_int sh_ipi_id; 211 } softhand_t; 212 213 typedef struct softcpu { 214 struct cpu_info *sc_cpu; 215 softint_t sc_int[SOFTINT_COUNT]; 216 softhand_t sc_hand[1]; 217 } softcpu_t; 218 219 static void softint_thread(void *); 220 221 u_int softint_bytes = 32768; 222 u_int softint_timing; 223 static u_int softint_max; 224 static kmutex_t softint_lock; 225 static void *softint_netisrs[NETISR_MAX]; 226 227 /* 228 * softint_init_isr: 229 * 230 * Initialize a single interrupt level for a single CPU. 231 */ 232 static void 233 softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level) 234 { 235 struct cpu_info *ci; 236 softint_t *si; 237 int error; 238 239 si = &sc->sc_int[level]; 240 ci = sc->sc_cpu; 241 si->si_cpu = ci; 242 243 SIMPLEQ_INIT(&si->si_q); 244 245 error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR | 246 KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp, 247 "soft%s/%u", desc, ci->ci_index); 248 if (error != 0) 249 panic("softint_init_isr: error %d", error); 250 251 snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc, 252 ci->ci_index); 253 evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL, 254 "softint", si->si_name); 255 snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u", 256 desc, ci->ci_index); 257 evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL, 258 "softint", si->si_name_block); 259 260 si->si_lwp->l_private = si; 261 softint_init_md(si->si_lwp, level, &si->si_machdep); 262 } 263 264 /* 265 * softint_init: 266 * 267 * Initialize per-CPU data structures. Called from mi_cpu_attach(). 268 */ 269 void 270 softint_init(struct cpu_info *ci) 271 { 272 static struct cpu_info *first; 273 softcpu_t *sc, *scfirst; 274 softhand_t *sh, *shmax; 275 276 if (first == NULL) { 277 /* Boot CPU. */ 278 first = ci; 279 mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE); 280 softint_bytes = round_page(softint_bytes); 281 softint_max = (softint_bytes - sizeof(softcpu_t)) / 282 sizeof(softhand_t); 283 } 284 285 /* Use uvm_km(9) for persistent, page-aligned allocation. */ 286 sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0, 287 UVM_KMF_WIRED | UVM_KMF_ZERO); 288 if (sc == NULL) 289 panic("softint_init_cpu: cannot allocate memory"); 290 291 ci->ci_data.cpu_softcpu = sc; 292 ci->ci_data.cpu_softints = 0; 293 sc->sc_cpu = ci; 294 295 softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET); 296 softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO); 297 softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK); 298 softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL); 299 300 if (first != ci) { 301 mutex_enter(&softint_lock); 302 scfirst = first->ci_data.cpu_softcpu; 303 sh = sc->sc_hand; 304 memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max); 305 /* Update pointers for this CPU. */ 306 for (shmax = sh + softint_max; sh < shmax; sh++) { 307 if (sh->sh_func == NULL) 308 continue; 309 sh->sh_isr = 310 &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK]; 311 } 312 mutex_exit(&softint_lock); 313 } else { 314 /* 315 * Establish handlers for legacy net interrupts. 316 * XXX Needs to go away. 317 */ 318 #define DONETISR(n, f) \ 319 softint_netisrs[(n)] = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,\ 320 (void (*)(void *))(f), NULL) 321 #include <net/netisr_dispatch.h> 322 } 323 } 324 325 /* 326 * softint_establish: 327 * 328 * Register a software interrupt handler. 329 */ 330 void * 331 softint_establish(u_int flags, void (*func)(void *), void *arg) 332 { 333 CPU_INFO_ITERATOR cii; 334 struct cpu_info *ci; 335 softcpu_t *sc; 336 softhand_t *sh; 337 u_int level, index; 338 u_int ipi_id = 0; 339 void *sih; 340 341 level = (flags & SOFTINT_LVLMASK); 342 KASSERT(level < SOFTINT_COUNT); 343 KASSERT((flags & SOFTINT_IMPMASK) == 0); 344 345 mutex_enter(&softint_lock); 346 347 /* Find a free slot. */ 348 sc = curcpu()->ci_data.cpu_softcpu; 349 for (index = 1; index < softint_max; index++) { 350 if (sc->sc_hand[index].sh_func == NULL) 351 break; 352 } 353 if (index == softint_max) { 354 mutex_exit(&softint_lock); 355 printf("WARNING: softint_establish: table full, " 356 "increase softint_bytes\n"); 357 return NULL; 358 } 359 sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc); 360 361 if (flags & SOFTINT_RCPU) { 362 if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) { 363 mutex_exit(&softint_lock); 364 return NULL; 365 } 366 } 367 368 /* Set up the handler on each CPU. */ 369 if (ncpu < 2) { 370 /* XXX hack for machines with no CPU_INFO_FOREACH() early on */ 371 sc = curcpu()->ci_data.cpu_softcpu; 372 sh = &sc->sc_hand[index]; 373 sh->sh_isr = &sc->sc_int[level]; 374 sh->sh_func = func; 375 sh->sh_arg = arg; 376 sh->sh_flags = flags; 377 sh->sh_ipi_id = ipi_id; 378 } else for (CPU_INFO_FOREACH(cii, ci)) { 379 sc = ci->ci_data.cpu_softcpu; 380 sh = &sc->sc_hand[index]; 381 sh->sh_isr = &sc->sc_int[level]; 382 sh->sh_func = func; 383 sh->sh_arg = arg; 384 sh->sh_flags = flags; 385 sh->sh_ipi_id = ipi_id; 386 } 387 mutex_exit(&softint_lock); 388 389 return sih; 390 } 391 392 /* 393 * softint_disestablish: 394 * 395 * Unregister a software interrupt handler. The soft interrupt could 396 * still be active at this point, but the caller commits not to try 397 * and trigger it again once this call is made. The caller must not 398 * hold any locks that could be taken from soft interrupt context, 399 * because we will wait for the softint to complete if it's still 400 * running. 401 */ 402 void 403 softint_disestablish(void *arg) 404 { 405 CPU_INFO_ITERATOR cii; 406 struct cpu_info *ci; 407 softcpu_t *sc; 408 softhand_t *sh; 409 uintptr_t offset; 410 uint64_t where; 411 u_int flags; 412 413 offset = (uintptr_t)arg; 414 KASSERTMSG(offset != 0 && offset < softint_bytes, "%"PRIuPTR" %u", 415 offset, softint_bytes); 416 417 /* 418 * Unregister an IPI handler if there is any. Note: there is 419 * no need to disable preemption here - ID is stable. 420 */ 421 sc = curcpu()->ci_data.cpu_softcpu; 422 sh = (softhand_t *)((uint8_t *)sc + offset); 423 if (sh->sh_ipi_id) { 424 ipi_unregister(sh->sh_ipi_id); 425 } 426 427 /* 428 * Run a cross call so we see up to date values of sh_flags from 429 * all CPUs. Once softint_disestablish() is called, the caller 430 * commits to not trigger the interrupt and set SOFTINT_ACTIVE on 431 * it again. So, we are only looking for handler records with 432 * SOFTINT_ACTIVE already set. 433 */ 434 if (__predict_true(mp_online)) { 435 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL); 436 xc_wait(where); 437 } 438 439 for (;;) { 440 /* Collect flag values from each CPU. */ 441 flags = 0; 442 for (CPU_INFO_FOREACH(cii, ci)) { 443 sc = ci->ci_data.cpu_softcpu; 444 sh = (softhand_t *)((uint8_t *)sc + offset); 445 KASSERT(sh->sh_func != NULL); 446 flags |= sh->sh_flags; 447 } 448 /* Inactive on all CPUs? */ 449 if ((flags & SOFTINT_ACTIVE) == 0) { 450 break; 451 } 452 /* Oops, still active. Wait for it to clear. */ 453 (void)kpause("softdis", false, 1, NULL); 454 } 455 456 /* Clear the handler on each CPU. */ 457 mutex_enter(&softint_lock); 458 for (CPU_INFO_FOREACH(cii, ci)) { 459 sc = ci->ci_data.cpu_softcpu; 460 sh = (softhand_t *)((uint8_t *)sc + offset); 461 KASSERT(sh->sh_func != NULL); 462 sh->sh_func = NULL; 463 } 464 mutex_exit(&softint_lock); 465 } 466 467 /* 468 * softint_schedule: 469 * 470 * Trigger a software interrupt. Must be called from a hardware 471 * interrupt handler, or with preemption disabled (since we are 472 * using the value of curcpu()). 473 */ 474 void 475 softint_schedule(void *arg) 476 { 477 softhand_t *sh; 478 softint_t *si; 479 uintptr_t offset; 480 int s; 481 482 KASSERT(kpreempt_disabled()); 483 484 /* Find the handler record for this CPU. */ 485 offset = (uintptr_t)arg; 486 KASSERTMSG(offset != 0 && offset < softint_bytes, "%"PRIuPTR" %u", 487 offset, softint_bytes); 488 sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset); 489 490 /* If it's already pending there's nothing to do. */ 491 if ((sh->sh_flags & SOFTINT_PENDING) != 0) { 492 return; 493 } 494 495 /* 496 * Enqueue the handler into the LWP's pending list. 497 * If the LWP is completely idle, then make it run. 498 */ 499 s = splhigh(); 500 if ((sh->sh_flags & SOFTINT_PENDING) == 0) { 501 si = sh->sh_isr; 502 sh->sh_flags |= SOFTINT_PENDING; 503 SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q); 504 if (si->si_active == 0) { 505 si->si_active = 1; 506 softint_trigger(si->si_machdep); 507 } 508 } 509 splx(s); 510 } 511 512 /* 513 * softint_schedule_cpu: 514 * 515 * Trigger a software interrupt on a target CPU. This invokes 516 * softint_schedule() for the local CPU or send an IPI to invoke 517 * this routine on the remote CPU. Preemption must be disabled. 518 */ 519 void 520 softint_schedule_cpu(void *arg, struct cpu_info *ci) 521 { 522 KASSERT(kpreempt_disabled()); 523 524 if (curcpu() != ci) { 525 const softcpu_t *sc = ci->ci_data.cpu_softcpu; 526 const uintptr_t offset = (uintptr_t)arg; 527 const softhand_t *sh; 528 529 sh = (const softhand_t *)((const uint8_t *)sc + offset); 530 KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0); 531 ipi_trigger(sh->sh_ipi_id, ci); 532 return; 533 } 534 535 /* Just a local CPU. */ 536 softint_schedule(arg); 537 } 538 539 /* 540 * softint_execute: 541 * 542 * Invoke handlers for the specified soft interrupt. 543 * Must be entered at splhigh. Will drop the priority 544 * to the level specified, but returns back at splhigh. 545 */ 546 static inline void 547 softint_execute(softint_t *si, lwp_t *l, int s) 548 { 549 softhand_t *sh; 550 bool havelock; 551 552 #ifdef __HAVE_FAST_SOFTINTS 553 KASSERT(si->si_lwp == curlwp); 554 #else 555 /* May be running in user context. */ 556 #endif 557 KASSERT(si->si_cpu == curcpu()); 558 KASSERT(si->si_lwp->l_wchan == NULL); 559 KASSERT(si->si_active); 560 561 havelock = false; 562 563 /* 564 * Note: due to priority inheritance we may have interrupted a 565 * higher priority LWP. Since the soft interrupt must be quick 566 * and is non-preemptable, we don't bother yielding. 567 */ 568 569 while (!SIMPLEQ_EMPTY(&si->si_q)) { 570 /* 571 * Pick the longest waiting handler to run. We block 572 * interrupts but do not lock in order to do this, as 573 * we are protecting against the local CPU only. 574 */ 575 sh = SIMPLEQ_FIRST(&si->si_q); 576 SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q); 577 KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0); 578 KASSERT((sh->sh_flags & SOFTINT_ACTIVE) == 0); 579 sh->sh_flags ^= (SOFTINT_PENDING | SOFTINT_ACTIVE); 580 splx(s); 581 582 /* Run the handler. */ 583 if (sh->sh_flags & SOFTINT_MPSAFE) { 584 if (havelock) { 585 KERNEL_UNLOCK_ONE(l); 586 havelock = false; 587 } 588 } else if (!havelock) { 589 KERNEL_LOCK(1, l); 590 havelock = true; 591 } 592 (*sh->sh_func)(sh->sh_arg); 593 594 /* Diagnostic: check that spin-locks have not leaked. */ 595 KASSERTMSG(curcpu()->ci_mtx_count == 0, 596 "%s: ci_mtx_count (%d) != 0, sh_func %p\n", 597 __func__, curcpu()->ci_mtx_count, sh->sh_func); 598 599 (void)splhigh(); 600 KASSERT((sh->sh_flags & SOFTINT_ACTIVE) != 0); 601 sh->sh_flags ^= SOFTINT_ACTIVE; 602 } 603 604 if (havelock) { 605 KERNEL_UNLOCK_ONE(l); 606 } 607 608 /* 609 * Unlocked, but only for statistics. 610 * Should be per-CPU to prevent cache ping-pong. 611 */ 612 curcpu()->ci_data.cpu_nsoft++; 613 614 KASSERT(si->si_cpu == curcpu()); 615 KASSERT(si->si_lwp->l_wchan == NULL); 616 KASSERT(si->si_active); 617 si->si_evcnt.ev_count++; 618 si->si_active = 0; 619 } 620 621 /* 622 * softint_block: 623 * 624 * Update statistics when the soft interrupt blocks. 625 */ 626 void 627 softint_block(lwp_t *l) 628 { 629 softint_t *si = l->l_private; 630 631 KASSERT((l->l_pflag & LP_INTR) != 0); 632 si->si_evcnt_block.ev_count++; 633 } 634 635 /* 636 * schednetisr: 637 * 638 * Trigger a legacy network interrupt. XXX Needs to go away. 639 */ 640 void 641 schednetisr(int isr) 642 { 643 644 softint_schedule(softint_netisrs[isr]); 645 } 646 647 #ifndef __HAVE_FAST_SOFTINTS 648 649 #ifdef __HAVE_PREEMPTION 650 #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS 651 #endif 652 653 /* 654 * softint_init_md: 655 * 656 * Slow path: perform machine-dependent initialization. 657 */ 658 void 659 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) 660 { 661 softint_t *si; 662 663 *machdep = (1 << level); 664 si = l->l_private; 665 666 lwp_lock(l); 667 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_mutex); 668 lwp_lock(l); 669 /* Cheat and make the KASSERT in softint_thread() happy. */ 670 si->si_active = 1; 671 l->l_stat = LSRUN; 672 sched_enqueue(l, false); 673 lwp_unlock(l); 674 } 675 676 /* 677 * softint_trigger: 678 * 679 * Slow path: cause a soft interrupt handler to begin executing. 680 * Called at IPL_HIGH. 681 */ 682 void 683 softint_trigger(uintptr_t machdep) 684 { 685 struct cpu_info *ci; 686 lwp_t *l; 687 688 l = curlwp; 689 ci = l->l_cpu; 690 ci->ci_data.cpu_softints |= machdep; 691 if (l == ci->ci_data.cpu_idlelwp) { 692 cpu_need_resched(ci, 0); 693 } else { 694 /* MI equivalent of aston() */ 695 cpu_signotify(l); 696 } 697 } 698 699 /* 700 * softint_thread: 701 * 702 * Slow path: MI software interrupt dispatch. 703 */ 704 void 705 softint_thread(void *cookie) 706 { 707 softint_t *si; 708 lwp_t *l; 709 int s; 710 711 l = curlwp; 712 si = l->l_private; 713 714 for (;;) { 715 /* 716 * Clear pending status and run it. We must drop the 717 * spl before mi_switch(), since IPL_HIGH may be higher 718 * than IPL_SCHED (and it is not safe to switch at a 719 * higher level). 720 */ 721 s = splhigh(); 722 l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep; 723 softint_execute(si, l, s); 724 splx(s); 725 726 lwp_lock(l); 727 l->l_stat = LSIDL; 728 mi_switch(l); 729 } 730 } 731 732 /* 733 * softint_picklwp: 734 * 735 * Slow path: called from mi_switch() to pick the highest priority 736 * soft interrupt LWP that needs to run. 737 */ 738 lwp_t * 739 softint_picklwp(void) 740 { 741 struct cpu_info *ci; 742 u_int mask; 743 softint_t *si; 744 lwp_t *l; 745 746 ci = curcpu(); 747 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 748 mask = ci->ci_data.cpu_softints; 749 750 if ((mask & (1 << SOFTINT_SERIAL)) != 0) { 751 l = si[SOFTINT_SERIAL].si_lwp; 752 } else if ((mask & (1 << SOFTINT_NET)) != 0) { 753 l = si[SOFTINT_NET].si_lwp; 754 } else if ((mask & (1 << SOFTINT_BIO)) != 0) { 755 l = si[SOFTINT_BIO].si_lwp; 756 } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) { 757 l = si[SOFTINT_CLOCK].si_lwp; 758 } else { 759 panic("softint_picklwp"); 760 } 761 762 return l; 763 } 764 765 /* 766 * softint_overlay: 767 * 768 * Slow path: called from lwp_userret() to run a soft interrupt 769 * within the context of a user thread. 770 */ 771 void 772 softint_overlay(void) 773 { 774 struct cpu_info *ci; 775 u_int softints, oflag; 776 softint_t *si; 777 pri_t obase; 778 lwp_t *l; 779 int s; 780 781 l = curlwp; 782 KASSERT((l->l_pflag & LP_INTR) == 0); 783 784 /* 785 * Arrange to elevate priority if the LWP blocks. Also, bind LWP 786 * to the CPU. Note: disable kernel preemption before doing that. 787 */ 788 s = splhigh(); 789 ci = l->l_cpu; 790 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 791 792 obase = l->l_kpribase; 793 l->l_kpribase = PRI_KERNEL_RT; 794 oflag = l->l_pflag; 795 l->l_pflag = oflag | LP_INTR | LP_BOUND; 796 797 while ((softints = ci->ci_data.cpu_softints) != 0) { 798 if ((softints & (1 << SOFTINT_SERIAL)) != 0) { 799 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_SERIAL); 800 softint_execute(&si[SOFTINT_SERIAL], l, s); 801 continue; 802 } 803 if ((softints & (1 << SOFTINT_NET)) != 0) { 804 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_NET); 805 softint_execute(&si[SOFTINT_NET], l, s); 806 continue; 807 } 808 if ((softints & (1 << SOFTINT_BIO)) != 0) { 809 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_BIO); 810 softint_execute(&si[SOFTINT_BIO], l, s); 811 continue; 812 } 813 if ((softints & (1 << SOFTINT_CLOCK)) != 0) { 814 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_CLOCK); 815 softint_execute(&si[SOFTINT_CLOCK], l, s); 816 continue; 817 } 818 } 819 l->l_pflag = oflag; 820 l->l_kpribase = obase; 821 splx(s); 822 } 823 824 #else /* !__HAVE_FAST_SOFTINTS */ 825 826 /* 827 * softint_thread: 828 * 829 * Fast path: the LWP is switched to without restoring any state, 830 * so we should not arrive here - there is a direct handoff between 831 * the interrupt stub and softint_dispatch(). 832 */ 833 void 834 softint_thread(void *cookie) 835 { 836 837 panic("softint_thread"); 838 } 839 840 /* 841 * softint_dispatch: 842 * 843 * Fast path: entry point from machine-dependent code. 844 */ 845 void 846 softint_dispatch(lwp_t *pinned, int s) 847 { 848 struct bintime now; 849 softint_t *si; 850 u_int timing; 851 lwp_t *l; 852 853 KASSERT((pinned->l_pflag & LP_RUNNING) != 0); 854 l = curlwp; 855 si = l->l_private; 856 857 /* 858 * Note the interrupted LWP, and mark the current LWP as running 859 * before proceeding. Although this must as a rule be done with 860 * the LWP locked, at this point no external agents will want to 861 * modify the interrupt LWP's state. 862 */ 863 timing = (softint_timing ? LP_TIMEINTR : 0); 864 l->l_switchto = pinned; 865 l->l_stat = LSONPROC; 866 l->l_pflag |= (LP_RUNNING | timing); 867 868 /* 869 * Dispatch the interrupt. If softints are being timed, charge 870 * for it. 871 */ 872 if (timing) 873 binuptime(&l->l_stime); 874 softint_execute(si, l, s); 875 if (timing) { 876 binuptime(&now); 877 updatertime(l, &now); 878 l->l_pflag &= ~LP_TIMEINTR; 879 } 880 881 /* Indicate a soft-interrupt switch. */ 882 pserialize_switchpoint(); 883 884 /* 885 * If we blocked while handling the interrupt, the pinned LWP is 886 * gone so switch to the idle LWP. It will select a new LWP to 887 * run. 888 * 889 * We must drop the priority level as switching at IPL_HIGH could 890 * deadlock the system. We have already set si->si_active = 0, 891 * which means another interrupt at this level can be triggered. 892 * That's not be a problem: we are lowering to level 's' which will 893 * prevent softint_dispatch() from being reentered at level 's', 894 * until the priority is finally dropped to IPL_NONE on entry to 895 * the LWP chosen by lwp_exit_switchaway(). 896 */ 897 l->l_stat = LSIDL; 898 if (l->l_switchto == NULL) { 899 splx(s); 900 pmap_deactivate(l); 901 lwp_exit_switchaway(l); 902 /* NOTREACHED */ 903 } 904 l->l_switchto = NULL; 905 l->l_pflag &= ~LP_RUNNING; 906 } 907 908 #endif /* !__HAVE_FAST_SOFTINTS */ 909