1 /* $NetBSD: kern_softint.c,v 1.15 2008/04/12 18:22:03 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Generic software interrupt framework. 41 * 42 * Overview 43 * 44 * The soft interrupt framework provides a mechanism to schedule a 45 * low priority callback that runs with thread context. It allows 46 * for dynamic registration of software interrupts, and for fair 47 * queueing and prioritization of those interrupts. The callbacks 48 * can be scheduled to run from nearly any point in the kernel: by 49 * code running with thread context, by code running from a 50 * hardware interrupt handler, and at any interrupt priority 51 * level. 52 * 53 * Priority levels 54 * 55 * Since soft interrupt dispatch can be tied to the underlying 56 * architecture's interrupt dispatch code, it can be limited 57 * both by the capabilities of the hardware and the capabilities 58 * of the interrupt dispatch code itself. The number of priority 59 * levels is restricted to four. In order of priority (lowest to 60 * highest) the levels are: clock, bio, net, serial. 61 * 62 * The names are symbolic and in isolation do not have any direct 63 * connection with a particular kind of device activity: they are 64 * only meant as a guide. 65 * 66 * The four priority levels map directly to scheduler priority 67 * levels, and where the architecture implements 'fast' software 68 * interrupts, they also map onto interrupt priorities. The 69 * interrupt priorities are intended to be hidden from machine 70 * independent code, which should use thread-safe mechanisms to 71 * synchronize with software interrupts (for example: mutexes). 72 * 73 * Capabilities 74 * 75 * Software interrupts run with limited machine context. In 76 * particular, they do not posess any address space context. They 77 * should not try to operate on user space addresses, or to use 78 * virtual memory facilities other than those noted as interrupt 79 * safe. 80 * 81 * Unlike hardware interrupts, software interrupts do have thread 82 * context. They may block on synchronization objects, sleep, and 83 * resume execution at a later time. 84 * 85 * Since software interrupts are a limited resource and run with 86 * higher priority than most other LWPs in the system, all 87 * block-and-resume activity by a software interrupt must be kept 88 * short to allow futher processing at that level to continue. By 89 * extension, code running with process context must take care to 90 * ensure that any lock that may be taken from a software interrupt 91 * can not be held for more than a short period of time. 92 * 93 * The kernel does not allow software interrupts to use facilities 94 * or perform actions that may block for a significant amount of 95 * time. This means that it's not valid for a software interrupt 96 * to sleep on condition variables or wait for resources to become 97 * available (for example, memory). 98 * 99 * Per-CPU operation 100 * 101 * If a soft interrupt is triggered on a CPU, it can only be 102 * dispatched on the same CPU. Each LWP dedicated to handling a 103 * soft interrupt is bound to its home CPU, so if the LWP blocks 104 * and needs to run again, it can only run there. Nearly all data 105 * structures used to manage software interrupts are per-CPU. 106 * 107 * The per-CPU requirement is intended to reduce "ping-pong" of 108 * cache lines between CPUs: lines occupied by data structures 109 * used to manage the soft interrupts, and lines occupied by data 110 * items being passed down to the soft interrupt. As a positive 111 * side effect, this also means that the soft interrupt dispatch 112 * code does not need to to use spinlocks to synchronize. 113 * 114 * Generic implementation 115 * 116 * A generic, low performance implementation is provided that 117 * works across all architectures, with no machine-dependent 118 * modifications needed. This implementation uses the scheduler, 119 * and so has a number of restrictions: 120 * 121 * 1) The software interrupts are not currently preemptive, so 122 * must wait for the currently executing LWP to yield the CPU. 123 * This can introduce latency. 124 * 125 * 2) An expensive context switch is required for a software 126 * interrupt to be handled. 127 * 128 * 'Fast' software interrupts 129 * 130 * If an architectures defines __HAVE_FAST_SOFTINTS, it implements 131 * the fast mechanism. Threads running either in the kernel or in 132 * userspace will be interrupted, but will not be preempted. When 133 * the soft interrupt completes execution, the interrupted LWP 134 * is resumed. Interrupt dispatch code must provide the minimum 135 * level of context necessary for the soft interrupt to block and 136 * be resumed at a later time. The machine-dependent dispatch 137 * path looks something like the following: 138 * 139 * softintr() 140 * { 141 * go to IPL_HIGH if necessary for switch; 142 * save any necessary registers in a format that can be 143 * restored by cpu_switchto if the softint blocks; 144 * arrange for cpu_switchto() to restore into the 145 * trampoline function; 146 * identify LWP to handle this interrupt; 147 * switch to the LWP's stack; 148 * switch register stacks, if necessary; 149 * assign new value of curlwp; 150 * call MI softint_dispatch, passing old curlwp and IPL 151 * to execute interrupt at; 152 * switch back to old stack; 153 * switch back to old register stack, if necessary; 154 * restore curlwp; 155 * return to interrupted LWP; 156 * } 157 * 158 * If the soft interrupt blocks, a trampoline function is returned 159 * to in the context of the interrupted LWP, as arranged for by 160 * softint(): 161 * 162 * softint_ret() 163 * { 164 * unlock soft interrupt LWP; 165 * resume interrupt processing, likely returning to 166 * interrupted LWP or dispatching another, different 167 * interrupt; 168 * } 169 * 170 * Once the soft interrupt has fired (and even if it has blocked), 171 * no further soft interrupts at that level will be triggered by 172 * MI code until the soft interrupt handler has ceased execution. 173 * If a soft interrupt handler blocks and is resumed, it resumes 174 * execution as a normal LWP (kthread) and gains VM context. Only 175 * when it has completed and is ready to fire again will it 176 * interrupt other threads. 177 * 178 * Future directions 179 * 180 * Provide a cheap way to direct software interrupts to remote 181 * CPUs. Provide a way to enqueue work items into the handler 182 * record, removing additional spl calls (see subr_workqueue.c). 183 */ 184 185 #include <sys/cdefs.h> 186 __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.15 2008/04/12 18:22:03 ad Exp $"); 187 188 #include <sys/param.h> 189 #include <sys/malloc.h> 190 #include <sys/proc.h> 191 #include <sys/intr.h> 192 #include <sys/mutex.h> 193 #include <sys/kthread.h> 194 #include <sys/evcnt.h> 195 #include <sys/cpu.h> 196 197 #include <net/netisr.h> 198 199 #include <uvm/uvm_extern.h> 200 201 /* This could overlap with signal info in struct lwp. */ 202 typedef struct softint { 203 SIMPLEQ_HEAD(, softhand) si_q; 204 struct lwp *si_lwp; 205 struct cpu_info *si_cpu; 206 uintptr_t si_machdep; 207 struct evcnt si_evcnt; 208 struct evcnt si_evcnt_block; 209 int si_active; 210 char si_name[8]; 211 char si_name_block[8+6]; 212 } softint_t; 213 214 typedef struct softhand { 215 SIMPLEQ_ENTRY(softhand) sh_q; 216 void (*sh_func)(void *); 217 void *sh_arg; 218 softint_t *sh_isr; 219 u_int sh_pending; 220 u_int sh_flags; 221 } softhand_t; 222 223 typedef struct softcpu { 224 struct cpu_info *sc_cpu; 225 softint_t sc_int[SOFTINT_COUNT]; 226 softhand_t sc_hand[1]; 227 } softcpu_t; 228 229 static void softint_thread(void *); 230 231 u_int softint_bytes = 8192; 232 u_int softint_timing; 233 static u_int softint_max; 234 static kmutex_t softint_lock; 235 static void *softint_netisrs[32]; 236 237 /* 238 * softint_init_isr: 239 * 240 * Initialize a single interrupt level for a single CPU. 241 */ 242 static void 243 softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level) 244 { 245 struct cpu_info *ci; 246 softint_t *si; 247 int error; 248 249 si = &sc->sc_int[level]; 250 ci = sc->sc_cpu; 251 si->si_cpu = ci; 252 253 SIMPLEQ_INIT(&si->si_q); 254 255 error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR | 256 KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp, 257 "soft%s/%u", desc, ci->ci_index); 258 if (error != 0) 259 panic("softint_init_isr: error %d", error); 260 261 snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc, 262 ci->ci_index); 263 evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_INTR, NULL, 264 "softint", si->si_name); 265 snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u", 266 desc, ci->ci_index); 267 evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_INTR, NULL, 268 "softint", si->si_name_block); 269 270 si->si_lwp->l_private = si; 271 softint_init_md(si->si_lwp, level, &si->si_machdep); 272 } 273 /* 274 * softint_init: 275 * 276 * Initialize per-CPU data structures. Called from mi_cpu_attach(). 277 */ 278 void 279 softint_init(struct cpu_info *ci) 280 { 281 static struct cpu_info *first; 282 softcpu_t *sc, *scfirst; 283 softhand_t *sh, *shmax; 284 285 if (first == NULL) { 286 /* Boot CPU. */ 287 first = ci; 288 mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE); 289 softint_bytes = round_page(softint_bytes); 290 softint_max = (softint_bytes - sizeof(softcpu_t)) / 291 sizeof(softhand_t); 292 } 293 294 sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0, 295 UVM_KMF_WIRED | UVM_KMF_ZERO); 296 if (sc == NULL) 297 panic("softint_init_cpu: cannot allocate memory"); 298 299 ci->ci_data.cpu_softcpu = sc; 300 ci->ci_data.cpu_softints = 0; 301 sc->sc_cpu = ci; 302 303 softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET); 304 softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO); 305 softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK); 306 softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL); 307 308 if (first != ci) { 309 mutex_enter(&softint_lock); 310 scfirst = first->ci_data.cpu_softcpu; 311 sh = sc->sc_hand; 312 memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max); 313 /* Update pointers for this CPU. */ 314 for (shmax = sh + softint_max; sh < shmax; sh++) { 315 if (sh->sh_func == NULL) 316 continue; 317 sh->sh_isr = 318 &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK]; 319 } 320 mutex_exit(&softint_lock); 321 } else { 322 /* 323 * Establish handlers for legacy net interrupts. 324 * XXX Needs to go away. 325 */ 326 #define DONETISR(n, f) \ 327 softint_netisrs[(n)] = \ 328 softint_establish(SOFTINT_NET, (void (*)(void *))(f), NULL) 329 #include <net/netisr_dispatch.h> 330 } 331 } 332 333 /* 334 * softint_establish: 335 * 336 * Register a software interrupt handler. 337 */ 338 void * 339 softint_establish(u_int flags, void (*func)(void *), void *arg) 340 { 341 CPU_INFO_ITERATOR cii; 342 struct cpu_info *ci; 343 softcpu_t *sc; 344 softhand_t *sh; 345 u_int level, index; 346 347 level = (flags & SOFTINT_LVLMASK); 348 KASSERT(level < SOFTINT_COUNT); 349 350 mutex_enter(&softint_lock); 351 352 /* Find a free slot. */ 353 sc = curcpu()->ci_data.cpu_softcpu; 354 for (index = 1; index < softint_max; index++) 355 if (sc->sc_hand[index].sh_func == NULL) 356 break; 357 if (index == softint_max) { 358 mutex_exit(&softint_lock); 359 printf("WARNING: softint_establish: table full, " 360 "increase softint_bytes\n"); 361 return NULL; 362 } 363 364 /* Set up the handler on each CPU. */ 365 if (ncpu < 2) { 366 /* XXX hack for machines with no CPU_INFO_FOREACH() early on */ 367 sc = curcpu()->ci_data.cpu_softcpu; 368 sh = &sc->sc_hand[index]; 369 sh->sh_isr = &sc->sc_int[level]; 370 sh->sh_func = func; 371 sh->sh_arg = arg; 372 sh->sh_flags = flags; 373 sh->sh_pending = 0; 374 } else for (CPU_INFO_FOREACH(cii, ci)) { 375 sc = ci->ci_data.cpu_softcpu; 376 sh = &sc->sc_hand[index]; 377 sh->sh_isr = &sc->sc_int[level]; 378 sh->sh_func = func; 379 sh->sh_arg = arg; 380 sh->sh_flags = flags; 381 sh->sh_pending = 0; 382 } 383 384 mutex_exit(&softint_lock); 385 386 return (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc); 387 } 388 389 /* 390 * softint_disestablish: 391 * 392 * Unregister a software interrupt handler. 393 */ 394 void 395 softint_disestablish(void *arg) 396 { 397 CPU_INFO_ITERATOR cii; 398 struct cpu_info *ci; 399 softcpu_t *sc; 400 softhand_t *sh; 401 uintptr_t offset; 402 403 offset = (uintptr_t)arg; 404 KASSERT(offset != 0 && offset < softint_bytes); 405 406 mutex_enter(&softint_lock); 407 408 /* Clear the handler on each CPU. */ 409 for (CPU_INFO_FOREACH(cii, ci)) { 410 sc = ci->ci_data.cpu_softcpu; 411 sh = (softhand_t *)((uint8_t *)sc + offset); 412 KASSERT(sh->sh_func != NULL); 413 KASSERT(sh->sh_pending == 0); 414 sh->sh_func = NULL; 415 } 416 417 mutex_exit(&softint_lock); 418 } 419 420 /* 421 * softint_schedule: 422 * 423 * Trigger a software interrupt. Must be called from a hardware 424 * interrupt handler, or with preemption disabled (since we are 425 * using the value of curcpu()). 426 */ 427 void 428 softint_schedule(void *arg) 429 { 430 softhand_t *sh; 431 softint_t *si; 432 uintptr_t offset; 433 int s; 434 435 /* Find the handler record for this CPU. */ 436 offset = (uintptr_t)arg; 437 KASSERT(offset != 0 && offset < softint_bytes); 438 sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset); 439 440 /* If it's already pending there's nothing to do. */ 441 if (sh->sh_pending) 442 return; 443 444 /* 445 * Enqueue the handler into the LWP's pending list. 446 * If the LWP is completely idle, then make it run. 447 */ 448 s = splhigh(); 449 if (!sh->sh_pending) { 450 si = sh->sh_isr; 451 sh->sh_pending = 1; 452 SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q); 453 if (si->si_active == 0) { 454 si->si_active = 1; 455 softint_trigger(si->si_machdep); 456 } 457 } 458 splx(s); 459 } 460 461 /* 462 * softint_execute: 463 * 464 * Invoke handlers for the specified soft interrupt. 465 * Must be entered at splhigh. Will drop the priority 466 * to the level specified, but returns back at splhigh. 467 */ 468 static inline void 469 softint_execute(softint_t *si, lwp_t *l, int s) 470 { 471 softhand_t *sh; 472 bool havelock; 473 474 #ifdef __HAVE_FAST_SOFTINTS 475 KASSERT(si->si_lwp == curlwp); 476 #else 477 /* May be running in user context. */ 478 #endif 479 KASSERT(si->si_cpu == curcpu()); 480 KASSERT(si->si_lwp->l_wchan == NULL); 481 KASSERT(si->si_active); 482 483 havelock = false; 484 485 /* 486 * Note: due to priority inheritance we may have interrupted a 487 * higher priority LWP. Since the soft interrupt must be quick 488 * and is non-preemptable, we don't bother yielding. 489 */ 490 491 while (!SIMPLEQ_EMPTY(&si->si_q)) { 492 /* 493 * Pick the longest waiting handler to run. We block 494 * interrupts but do not lock in order to do this, as 495 * we are protecting against the local CPU only. 496 */ 497 sh = SIMPLEQ_FIRST(&si->si_q); 498 SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q); 499 sh->sh_pending = 0; 500 splx(s); 501 502 /* Run the handler. */ 503 if ((sh->sh_flags & SOFTINT_MPSAFE) == 0 && !havelock) { 504 KERNEL_LOCK(1, l); 505 havelock = true; 506 } 507 (*sh->sh_func)(sh->sh_arg); 508 509 (void)splhigh(); 510 } 511 512 if (havelock) { 513 KERNEL_UNLOCK_ONE(l); 514 } 515 516 /* 517 * Unlocked, but only for statistics. 518 * Should be per-CPU to prevent cache ping-pong. 519 */ 520 uvmexp.softs++; 521 522 KASSERT(si->si_cpu == curcpu()); 523 KASSERT(si->si_lwp->l_wchan == NULL); 524 KASSERT(si->si_active); 525 si->si_evcnt.ev_count++; 526 si->si_active = 0; 527 } 528 529 /* 530 * softint_block: 531 * 532 * Update statistics when the soft interrupt blocks. 533 */ 534 void 535 softint_block(lwp_t *l) 536 { 537 softint_t *si = l->l_private; 538 539 KASSERT((l->l_pflag & LP_INTR) != 0); 540 si->si_evcnt_block.ev_count++; 541 } 542 543 /* 544 * schednetisr: 545 * 546 * Trigger a legacy network interrupt. XXX Needs to go away. 547 */ 548 void 549 schednetisr(int isr) 550 { 551 552 softint_schedule(softint_netisrs[isr]); 553 } 554 555 #ifndef __HAVE_FAST_SOFTINTS 556 557 /* 558 * softint_init_md: 559 * 560 * Slow path: perform machine-dependent initialization. 561 */ 562 void 563 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) 564 { 565 softint_t *si; 566 567 *machdep = (1 << level); 568 si = l->l_private; 569 570 lwp_lock(l); 571 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_mutex); 572 lwp_lock(l); 573 /* Cheat and make the KASSERT in softint_thread() happy. */ 574 si->si_active = 1; 575 l->l_stat = LSRUN; 576 sched_enqueue(l, false); 577 lwp_unlock(l); 578 } 579 580 /* 581 * softint_trigger: 582 * 583 * Slow path: cause a soft interrupt handler to begin executing. 584 * Called at IPL_HIGH. 585 */ 586 void 587 softint_trigger(uintptr_t machdep) 588 { 589 struct cpu_info *ci; 590 lwp_t *l; 591 592 l = curlwp; 593 ci = l->l_cpu; 594 ci->ci_data.cpu_softints |= machdep; 595 if (l == ci->ci_data.cpu_idlelwp) { 596 cpu_need_resched(ci, 0); 597 } else { 598 /* MI equivalent of aston() */ 599 cpu_signotify(l); 600 } 601 } 602 603 /* 604 * softint_thread: 605 * 606 * Slow path: MI software interrupt dispatch. 607 */ 608 void 609 softint_thread(void *cookie) 610 { 611 softint_t *si; 612 lwp_t *l; 613 int s; 614 615 l = curlwp; 616 si = l->l_private; 617 618 for (;;) { 619 /* 620 * Clear pending status and run it. We must drop the 621 * spl before mi_switch(), since IPL_HIGH may be higher 622 * than IPL_SCHED (and it is not safe to switch at a 623 * higher level). 624 */ 625 s = splhigh(); 626 l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep; 627 softint_execute(si, l, s); 628 splx(s); 629 630 lwp_lock(l); 631 l->l_stat = LSIDL; 632 mi_switch(l); 633 } 634 } 635 636 /* 637 * softint_picklwp: 638 * 639 * Slow path: called from mi_switch() to pick the highest priority 640 * soft interrupt LWP that needs to run. 641 */ 642 lwp_t * 643 softint_picklwp(void) 644 { 645 struct cpu_info *ci; 646 u_int mask; 647 softint_t *si; 648 lwp_t *l; 649 650 ci = curcpu(); 651 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 652 mask = ci->ci_data.cpu_softints; 653 654 if ((mask & (1 << SOFTINT_SERIAL)) != 0) { 655 l = si[SOFTINT_SERIAL].si_lwp; 656 } else if ((mask & (1 << SOFTINT_NET)) != 0) { 657 l = si[SOFTINT_NET].si_lwp; 658 } else if ((mask & (1 << SOFTINT_BIO)) != 0) { 659 l = si[SOFTINT_BIO].si_lwp; 660 } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) { 661 l = si[SOFTINT_CLOCK].si_lwp; 662 } else { 663 panic("softint_picklwp"); 664 } 665 666 return l; 667 } 668 669 /* 670 * softint_overlay: 671 * 672 * Slow path: called from lwp_userret() to run a soft interrupt 673 * within the context of a user thread. 674 */ 675 void 676 softint_overlay(void) 677 { 678 struct cpu_info *ci; 679 u_int softints, oflag; 680 softint_t *si; 681 pri_t obase; 682 lwp_t *l; 683 int s; 684 685 l = curlwp; 686 ci = l->l_cpu; 687 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 688 689 KASSERT((l->l_pflag & LP_INTR) == 0); 690 691 /* Arrange to elevate priority if the LWP blocks. */ 692 s = splhigh(); 693 obase = l->l_kpribase; 694 l->l_kpribase = PRI_KERNEL_RT; 695 oflag = l->l_pflag; 696 l->l_pflag = oflag | LP_INTR | LP_BOUND; 697 while ((softints = ci->ci_data.cpu_softints) != 0) { 698 if ((softints & (1 << SOFTINT_SERIAL)) != 0) { 699 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_SERIAL); 700 softint_execute(&si[SOFTINT_SERIAL], l, s); 701 continue; 702 } 703 if ((softints & (1 << SOFTINT_NET)) != 0) { 704 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_NET); 705 softint_execute(&si[SOFTINT_NET], l, s); 706 continue; 707 } 708 if ((softints & (1 << SOFTINT_BIO)) != 0) { 709 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_BIO); 710 softint_execute(&si[SOFTINT_BIO], l, s); 711 continue; 712 } 713 if ((softints & (1 << SOFTINT_CLOCK)) != 0) { 714 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_CLOCK); 715 softint_execute(&si[SOFTINT_CLOCK], l, s); 716 continue; 717 } 718 } 719 l->l_pflag = oflag; 720 l->l_kpribase = obase; 721 splx(s); 722 } 723 724 #else /* !__HAVE_FAST_SOFTINTS */ 725 726 /* 727 * softint_thread: 728 * 729 * Fast path: the LWP is switched to without restoring any state, 730 * so we should not arrive here - there is a direct handoff between 731 * the interrupt stub and softint_dispatch(). 732 */ 733 void 734 softint_thread(void *cookie) 735 { 736 737 panic("softint_thread"); 738 } 739 740 /* 741 * softint_dispatch: 742 * 743 * Fast path: entry point from machine-dependent code. 744 */ 745 void 746 softint_dispatch(lwp_t *pinned, int s) 747 { 748 struct bintime now; 749 softint_t *si; 750 u_int timing; 751 lwp_t *l; 752 753 l = curlwp; 754 si = l->l_private; 755 756 /* 757 * Note the interrupted LWP, and mark the current LWP as running 758 * before proceeding. Although this must as a rule be done with 759 * the LWP locked, at this point no external agents will want to 760 * modify the interrupt LWP's state. 761 */ 762 timing = (softint_timing ? LW_TIMEINTR : 0); 763 l->l_switchto = pinned; 764 l->l_stat = LSONPROC; 765 l->l_flag |= (LW_RUNNING | timing); 766 767 /* 768 * Dispatch the interrupt. If softints are being timed, charge 769 * for it. 770 */ 771 if (timing) 772 binuptime(&l->l_stime); 773 softint_execute(si, l, s); 774 if (timing) { 775 binuptime(&now); 776 updatertime(l, &now); 777 l->l_flag &= ~LW_TIMEINTR; 778 } 779 780 /* 781 * If we blocked while handling the interrupt, the pinned LWP is 782 * gone so switch to the idle LWP. It will select a new LWP to 783 * run. 784 * 785 * We must drop the priority level as switching at IPL_HIGH could 786 * deadlock the system. We have already set si->si_active = 0, 787 * which means another interrupt at this level can be triggered. 788 * That's not be a problem: we are lowering to level 's' which will 789 * prevent softint_dispatch() from being reentered at level 's', 790 * until the priority is finally dropped to IPL_NONE on entry to 791 * the idle loop. 792 */ 793 l->l_stat = LSIDL; 794 if (l->l_switchto == NULL) { 795 splx(s); 796 pmap_deactivate(l); 797 lwp_exit_switchaway(l); 798 /* NOTREACHED */ 799 } 800 l->l_switchto = NULL; 801 l->l_flag &= ~LW_RUNNING; 802 } 803 804 #endif /* !__HAVE_FAST_SOFTINTS */ 805