1 /* $NetBSD: kern_softint.c,v 1.20 2008/04/28 23:00:22 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Generic software interrupt framework. 34 * 35 * Overview 36 * 37 * The soft interrupt framework provides a mechanism to schedule a 38 * low priority callback that runs with thread context. It allows 39 * for dynamic registration of software interrupts, and for fair 40 * queueing and prioritization of those interrupts. The callbacks 41 * can be scheduled to run from nearly any point in the kernel: by 42 * code running with thread context, by code running from a 43 * hardware interrupt handler, and at any interrupt priority 44 * level. 45 * 46 * Priority levels 47 * 48 * Since soft interrupt dispatch can be tied to the underlying 49 * architecture's interrupt dispatch code, it can be limited 50 * both by the capabilities of the hardware and the capabilities 51 * of the interrupt dispatch code itself. The number of priority 52 * levels is restricted to four. In order of priority (lowest to 53 * highest) the levels are: clock, bio, net, serial. 54 * 55 * The names are symbolic and in isolation do not have any direct 56 * connection with a particular kind of device activity: they are 57 * only meant as a guide. 58 * 59 * The four priority levels map directly to scheduler priority 60 * levels, and where the architecture implements 'fast' software 61 * interrupts, they also map onto interrupt priorities. The 62 * interrupt priorities are intended to be hidden from machine 63 * independent code, which should use thread-safe mechanisms to 64 * synchronize with software interrupts (for example: mutexes). 65 * 66 * Capabilities 67 * 68 * Software interrupts run with limited machine context. In 69 * particular, they do not posess any address space context. They 70 * should not try to operate on user space addresses, or to use 71 * virtual memory facilities other than those noted as interrupt 72 * safe. 73 * 74 * Unlike hardware interrupts, software interrupts do have thread 75 * context. They may block on synchronization objects, sleep, and 76 * resume execution at a later time. 77 * 78 * Since software interrupts are a limited resource and run with 79 * higher priority than most other LWPs in the system, all 80 * block-and-resume activity by a software interrupt must be kept 81 * short to allow futher processing at that level to continue. By 82 * extension, code running with process context must take care to 83 * ensure that any lock that may be taken from a software interrupt 84 * can not be held for more than a short period of time. 85 * 86 * The kernel does not allow software interrupts to use facilities 87 * or perform actions that may block for a significant amount of 88 * time. This means that it's not valid for a software interrupt 89 * to sleep on condition variables or wait for resources to become 90 * available (for example, memory). 91 * 92 * Per-CPU operation 93 * 94 * If a soft interrupt is triggered on a CPU, it can only be 95 * dispatched on the same CPU. Each LWP dedicated to handling a 96 * soft interrupt is bound to its home CPU, so if the LWP blocks 97 * and needs to run again, it can only run there. Nearly all data 98 * structures used to manage software interrupts are per-CPU. 99 * 100 * The per-CPU requirement is intended to reduce "ping-pong" of 101 * cache lines between CPUs: lines occupied by data structures 102 * used to manage the soft interrupts, and lines occupied by data 103 * items being passed down to the soft interrupt. As a positive 104 * side effect, this also means that the soft interrupt dispatch 105 * code does not need to to use spinlocks to synchronize. 106 * 107 * Generic implementation 108 * 109 * A generic, low performance implementation is provided that 110 * works across all architectures, with no machine-dependent 111 * modifications needed. This implementation uses the scheduler, 112 * and so has a number of restrictions: 113 * 114 * 1) The software interrupts are not currently preemptive, so 115 * must wait for the currently executing LWP to yield the CPU. 116 * This can introduce latency. 117 * 118 * 2) An expensive context switch is required for a software 119 * interrupt to be handled. 120 * 121 * 'Fast' software interrupts 122 * 123 * If an architectures defines __HAVE_FAST_SOFTINTS, it implements 124 * the fast mechanism. Threads running either in the kernel or in 125 * userspace will be interrupted, but will not be preempted. When 126 * the soft interrupt completes execution, the interrupted LWP 127 * is resumed. Interrupt dispatch code must provide the minimum 128 * level of context necessary for the soft interrupt to block and 129 * be resumed at a later time. The machine-dependent dispatch 130 * path looks something like the following: 131 * 132 * softintr() 133 * { 134 * go to IPL_HIGH if necessary for switch; 135 * save any necessary registers in a format that can be 136 * restored by cpu_switchto if the softint blocks; 137 * arrange for cpu_switchto() to restore into the 138 * trampoline function; 139 * identify LWP to handle this interrupt; 140 * switch to the LWP's stack; 141 * switch register stacks, if necessary; 142 * assign new value of curlwp; 143 * call MI softint_dispatch, passing old curlwp and IPL 144 * to execute interrupt at; 145 * switch back to old stack; 146 * switch back to old register stack, if necessary; 147 * restore curlwp; 148 * return to interrupted LWP; 149 * } 150 * 151 * If the soft interrupt blocks, a trampoline function is returned 152 * to in the context of the interrupted LWP, as arranged for by 153 * softint(): 154 * 155 * softint_ret() 156 * { 157 * unlock soft interrupt LWP; 158 * resume interrupt processing, likely returning to 159 * interrupted LWP or dispatching another, different 160 * interrupt; 161 * } 162 * 163 * Once the soft interrupt has fired (and even if it has blocked), 164 * no further soft interrupts at that level will be triggered by 165 * MI code until the soft interrupt handler has ceased execution. 166 * If a soft interrupt handler blocks and is resumed, it resumes 167 * execution as a normal LWP (kthread) and gains VM context. Only 168 * when it has completed and is ready to fire again will it 169 * interrupt other threads. 170 * 171 * Future directions 172 * 173 * Provide a cheap way to direct software interrupts to remote 174 * CPUs. Provide a way to enqueue work items into the handler 175 * record, removing additional spl calls (see subr_workqueue.c). 176 */ 177 178 #include <sys/cdefs.h> 179 __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.20 2008/04/28 23:00:22 ad Exp $"); 180 181 #include <sys/param.h> 182 #include <sys/malloc.h> 183 #include <sys/proc.h> 184 #include <sys/intr.h> 185 #include <sys/mutex.h> 186 #include <sys/kthread.h> 187 #include <sys/evcnt.h> 188 #include <sys/cpu.h> 189 190 #include <net/netisr.h> 191 192 #include <uvm/uvm_extern.h> 193 194 /* This could overlap with signal info in struct lwp. */ 195 typedef struct softint { 196 SIMPLEQ_HEAD(, softhand) si_q; 197 struct lwp *si_lwp; 198 struct cpu_info *si_cpu; 199 uintptr_t si_machdep; 200 struct evcnt si_evcnt; 201 struct evcnt si_evcnt_block; 202 int si_active; 203 char si_name[8]; 204 char si_name_block[8+6]; 205 } softint_t; 206 207 typedef struct softhand { 208 SIMPLEQ_ENTRY(softhand) sh_q; 209 void (*sh_func)(void *); 210 void *sh_arg; 211 softint_t *sh_isr; 212 u_int sh_pending; 213 u_int sh_flags; 214 } softhand_t; 215 216 typedef struct softcpu { 217 struct cpu_info *sc_cpu; 218 softint_t sc_int[SOFTINT_COUNT]; 219 softhand_t sc_hand[1]; 220 } softcpu_t; 221 222 static void softint_thread(void *); 223 224 u_int softint_bytes = 8192; 225 u_int softint_timing; 226 static u_int softint_max; 227 static kmutex_t softint_lock; 228 static void *softint_netisrs[32]; 229 230 /* 231 * softint_init_isr: 232 * 233 * Initialize a single interrupt level for a single CPU. 234 */ 235 static void 236 softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level) 237 { 238 struct cpu_info *ci; 239 softint_t *si; 240 int error; 241 242 si = &sc->sc_int[level]; 243 ci = sc->sc_cpu; 244 si->si_cpu = ci; 245 246 SIMPLEQ_INIT(&si->si_q); 247 248 error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR | 249 KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp, 250 "soft%s/%u", desc, ci->ci_index); 251 if (error != 0) 252 panic("softint_init_isr: error %d", error); 253 254 snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc, 255 ci->ci_index); 256 evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL, 257 "softint", si->si_name); 258 snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u", 259 desc, ci->ci_index); 260 evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL, 261 "softint", si->si_name_block); 262 263 si->si_lwp->l_private = si; 264 softint_init_md(si->si_lwp, level, &si->si_machdep); 265 } 266 /* 267 * softint_init: 268 * 269 * Initialize per-CPU data structures. Called from mi_cpu_attach(). 270 */ 271 void 272 softint_init(struct cpu_info *ci) 273 { 274 static struct cpu_info *first; 275 softcpu_t *sc, *scfirst; 276 softhand_t *sh, *shmax; 277 278 if (first == NULL) { 279 /* Boot CPU. */ 280 first = ci; 281 mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE); 282 softint_bytes = round_page(softint_bytes); 283 softint_max = (softint_bytes - sizeof(softcpu_t)) / 284 sizeof(softhand_t); 285 } 286 287 sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0, 288 UVM_KMF_WIRED | UVM_KMF_ZERO); 289 if (sc == NULL) 290 panic("softint_init_cpu: cannot allocate memory"); 291 292 ci->ci_data.cpu_softcpu = sc; 293 ci->ci_data.cpu_softints = 0; 294 sc->sc_cpu = ci; 295 296 softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET); 297 softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO); 298 softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK); 299 softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL); 300 301 if (first != ci) { 302 mutex_enter(&softint_lock); 303 scfirst = first->ci_data.cpu_softcpu; 304 sh = sc->sc_hand; 305 memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max); 306 /* Update pointers for this CPU. */ 307 for (shmax = sh + softint_max; sh < shmax; sh++) { 308 if (sh->sh_func == NULL) 309 continue; 310 sh->sh_isr = 311 &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK]; 312 } 313 mutex_exit(&softint_lock); 314 } else { 315 /* 316 * Establish handlers for legacy net interrupts. 317 * XXX Needs to go away. 318 */ 319 #define DONETISR(n, f) \ 320 softint_netisrs[(n)] = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,\ 321 (void (*)(void *))(f), NULL) 322 #include <net/netisr_dispatch.h> 323 } 324 } 325 326 /* 327 * softint_establish: 328 * 329 * Register a software interrupt handler. 330 */ 331 void * 332 softint_establish(u_int flags, void (*func)(void *), void *arg) 333 { 334 CPU_INFO_ITERATOR cii; 335 struct cpu_info *ci; 336 softcpu_t *sc; 337 softhand_t *sh; 338 u_int level, index; 339 340 level = (flags & SOFTINT_LVLMASK); 341 KASSERT(level < SOFTINT_COUNT); 342 343 mutex_enter(&softint_lock); 344 345 /* Find a free slot. */ 346 sc = curcpu()->ci_data.cpu_softcpu; 347 for (index = 1; index < softint_max; index++) 348 if (sc->sc_hand[index].sh_func == NULL) 349 break; 350 if (index == softint_max) { 351 mutex_exit(&softint_lock); 352 printf("WARNING: softint_establish: table full, " 353 "increase softint_bytes\n"); 354 return NULL; 355 } 356 357 /* Set up the handler on each CPU. */ 358 if (ncpu < 2) { 359 /* XXX hack for machines with no CPU_INFO_FOREACH() early on */ 360 sc = curcpu()->ci_data.cpu_softcpu; 361 sh = &sc->sc_hand[index]; 362 sh->sh_isr = &sc->sc_int[level]; 363 sh->sh_func = func; 364 sh->sh_arg = arg; 365 sh->sh_flags = flags; 366 sh->sh_pending = 0; 367 } else for (CPU_INFO_FOREACH(cii, ci)) { 368 sc = ci->ci_data.cpu_softcpu; 369 sh = &sc->sc_hand[index]; 370 sh->sh_isr = &sc->sc_int[level]; 371 sh->sh_func = func; 372 sh->sh_arg = arg; 373 sh->sh_flags = flags; 374 sh->sh_pending = 0; 375 } 376 377 mutex_exit(&softint_lock); 378 379 return (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc); 380 } 381 382 /* 383 * softint_disestablish: 384 * 385 * Unregister a software interrupt handler. 386 */ 387 void 388 softint_disestablish(void *arg) 389 { 390 CPU_INFO_ITERATOR cii; 391 struct cpu_info *ci; 392 softcpu_t *sc; 393 softhand_t *sh; 394 uintptr_t offset; 395 396 offset = (uintptr_t)arg; 397 KASSERT(offset != 0 && offset < softint_bytes); 398 399 mutex_enter(&softint_lock); 400 401 /* Clear the handler on each CPU. */ 402 for (CPU_INFO_FOREACH(cii, ci)) { 403 sc = ci->ci_data.cpu_softcpu; 404 sh = (softhand_t *)((uint8_t *)sc + offset); 405 KASSERT(sh->sh_func != NULL); 406 KASSERT(sh->sh_pending == 0); 407 sh->sh_func = NULL; 408 } 409 410 mutex_exit(&softint_lock); 411 } 412 413 /* 414 * softint_schedule: 415 * 416 * Trigger a software interrupt. Must be called from a hardware 417 * interrupt handler, or with preemption disabled (since we are 418 * using the value of curcpu()). 419 */ 420 void 421 softint_schedule(void *arg) 422 { 423 softhand_t *sh; 424 softint_t *si; 425 uintptr_t offset; 426 int s; 427 428 KASSERT(kpreempt_disabled()); 429 430 /* Find the handler record for this CPU. */ 431 offset = (uintptr_t)arg; 432 KASSERT(offset != 0 && offset < softint_bytes); 433 sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset); 434 435 /* If it's already pending there's nothing to do. */ 436 if (sh->sh_pending) 437 return; 438 439 /* 440 * Enqueue the handler into the LWP's pending list. 441 * If the LWP is completely idle, then make it run. 442 */ 443 s = splhigh(); 444 if (!sh->sh_pending) { 445 si = sh->sh_isr; 446 sh->sh_pending = 1; 447 SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q); 448 if (si->si_active == 0) { 449 si->si_active = 1; 450 softint_trigger(si->si_machdep); 451 } 452 } 453 splx(s); 454 } 455 456 /* 457 * softint_execute: 458 * 459 * Invoke handlers for the specified soft interrupt. 460 * Must be entered at splhigh. Will drop the priority 461 * to the level specified, but returns back at splhigh. 462 */ 463 static inline void 464 softint_execute(softint_t *si, lwp_t *l, int s) 465 { 466 softhand_t *sh; 467 bool havelock; 468 469 #ifdef __HAVE_FAST_SOFTINTS 470 KASSERT(si->si_lwp == curlwp); 471 #else 472 /* May be running in user context. */ 473 #endif 474 KASSERT(si->si_cpu == curcpu()); 475 KASSERT(si->si_lwp->l_wchan == NULL); 476 KASSERT(si->si_active); 477 478 havelock = false; 479 480 /* 481 * Note: due to priority inheritance we may have interrupted a 482 * higher priority LWP. Since the soft interrupt must be quick 483 * and is non-preemptable, we don't bother yielding. 484 */ 485 486 while (!SIMPLEQ_EMPTY(&si->si_q)) { 487 /* 488 * Pick the longest waiting handler to run. We block 489 * interrupts but do not lock in order to do this, as 490 * we are protecting against the local CPU only. 491 */ 492 sh = SIMPLEQ_FIRST(&si->si_q); 493 SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q); 494 sh->sh_pending = 0; 495 splx(s); 496 497 /* Run the handler. */ 498 if ((sh->sh_flags & SOFTINT_MPSAFE) == 0 && !havelock) { 499 KERNEL_LOCK(1, l); 500 havelock = true; 501 } 502 (*sh->sh_func)(sh->sh_arg); 503 504 (void)splhigh(); 505 } 506 507 if (havelock) { 508 KERNEL_UNLOCK_ONE(l); 509 } 510 511 /* 512 * Unlocked, but only for statistics. 513 * Should be per-CPU to prevent cache ping-pong. 514 */ 515 uvmexp.softs++; 516 517 KASSERT(si->si_cpu == curcpu()); 518 KASSERT(si->si_lwp->l_wchan == NULL); 519 KASSERT(si->si_active); 520 si->si_evcnt.ev_count++; 521 si->si_active = 0; 522 } 523 524 /* 525 * softint_block: 526 * 527 * Update statistics when the soft interrupt blocks. 528 */ 529 void 530 softint_block(lwp_t *l) 531 { 532 softint_t *si = l->l_private; 533 534 KASSERT((l->l_pflag & LP_INTR) != 0); 535 si->si_evcnt_block.ev_count++; 536 } 537 538 /* 539 * schednetisr: 540 * 541 * Trigger a legacy network interrupt. XXX Needs to go away. 542 */ 543 void 544 schednetisr(int isr) 545 { 546 547 softint_schedule(softint_netisrs[isr]); 548 } 549 550 #ifndef __HAVE_FAST_SOFTINTS 551 552 #ifdef __HAVE_PREEMPTION 553 #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS 554 #endif 555 556 /* 557 * softint_init_md: 558 * 559 * Slow path: perform machine-dependent initialization. 560 */ 561 void 562 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep) 563 { 564 softint_t *si; 565 566 *machdep = (1 << level); 567 si = l->l_private; 568 569 lwp_lock(l); 570 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_mutex); 571 lwp_lock(l); 572 /* Cheat and make the KASSERT in softint_thread() happy. */ 573 si->si_active = 1; 574 l->l_stat = LSRUN; 575 sched_enqueue(l, false); 576 lwp_unlock(l); 577 } 578 579 /* 580 * softint_trigger: 581 * 582 * Slow path: cause a soft interrupt handler to begin executing. 583 * Called at IPL_HIGH. 584 */ 585 void 586 softint_trigger(uintptr_t machdep) 587 { 588 struct cpu_info *ci; 589 lwp_t *l; 590 591 l = curlwp; 592 ci = l->l_cpu; 593 ci->ci_data.cpu_softints |= machdep; 594 if (l == ci->ci_data.cpu_idlelwp) { 595 cpu_need_resched(ci, 0); 596 } else { 597 /* MI equivalent of aston() */ 598 cpu_signotify(l); 599 } 600 } 601 602 /* 603 * softint_thread: 604 * 605 * Slow path: MI software interrupt dispatch. 606 */ 607 void 608 softint_thread(void *cookie) 609 { 610 softint_t *si; 611 lwp_t *l; 612 int s; 613 614 l = curlwp; 615 si = l->l_private; 616 617 for (;;) { 618 /* 619 * Clear pending status and run it. We must drop the 620 * spl before mi_switch(), since IPL_HIGH may be higher 621 * than IPL_SCHED (and it is not safe to switch at a 622 * higher level). 623 */ 624 s = splhigh(); 625 l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep; 626 softint_execute(si, l, s); 627 splx(s); 628 629 lwp_lock(l); 630 l->l_stat = LSIDL; 631 mi_switch(l); 632 } 633 } 634 635 /* 636 * softint_picklwp: 637 * 638 * Slow path: called from mi_switch() to pick the highest priority 639 * soft interrupt LWP that needs to run. 640 */ 641 lwp_t * 642 softint_picklwp(void) 643 { 644 struct cpu_info *ci; 645 u_int mask; 646 softint_t *si; 647 lwp_t *l; 648 649 ci = curcpu(); 650 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 651 mask = ci->ci_data.cpu_softints; 652 653 if ((mask & (1 << SOFTINT_SERIAL)) != 0) { 654 l = si[SOFTINT_SERIAL].si_lwp; 655 } else if ((mask & (1 << SOFTINT_NET)) != 0) { 656 l = si[SOFTINT_NET].si_lwp; 657 } else if ((mask & (1 << SOFTINT_BIO)) != 0) { 658 l = si[SOFTINT_BIO].si_lwp; 659 } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) { 660 l = si[SOFTINT_CLOCK].si_lwp; 661 } else { 662 panic("softint_picklwp"); 663 } 664 665 return l; 666 } 667 668 /* 669 * softint_overlay: 670 * 671 * Slow path: called from lwp_userret() to run a soft interrupt 672 * within the context of a user thread. 673 */ 674 void 675 softint_overlay(void) 676 { 677 struct cpu_info *ci; 678 u_int softints, oflag; 679 softint_t *si; 680 pri_t obase; 681 lwp_t *l; 682 int s; 683 684 l = curlwp; 685 ci = l->l_cpu; 686 si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int; 687 688 KASSERT((l->l_pflag & LP_INTR) == 0); 689 690 /* Arrange to elevate priority if the LWP blocks. */ 691 s = splhigh(); 692 obase = l->l_kpribase; 693 l->l_kpribase = PRI_KERNEL_RT; 694 oflag = l->l_pflag; 695 l->l_pflag = oflag | LP_INTR | LP_BOUND; 696 while ((softints = ci->ci_data.cpu_softints) != 0) { 697 if ((softints & (1 << SOFTINT_SERIAL)) != 0) { 698 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_SERIAL); 699 softint_execute(&si[SOFTINT_SERIAL], l, s); 700 continue; 701 } 702 if ((softints & (1 << SOFTINT_NET)) != 0) { 703 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_NET); 704 softint_execute(&si[SOFTINT_NET], l, s); 705 continue; 706 } 707 if ((softints & (1 << SOFTINT_BIO)) != 0) { 708 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_BIO); 709 softint_execute(&si[SOFTINT_BIO], l, s); 710 continue; 711 } 712 if ((softints & (1 << SOFTINT_CLOCK)) != 0) { 713 ci->ci_data.cpu_softints &= ~(1 << SOFTINT_CLOCK); 714 softint_execute(&si[SOFTINT_CLOCK], l, s); 715 continue; 716 } 717 } 718 l->l_pflag = oflag; 719 l->l_kpribase = obase; 720 splx(s); 721 } 722 723 #else /* !__HAVE_FAST_SOFTINTS */ 724 725 /* 726 * softint_thread: 727 * 728 * Fast path: the LWP is switched to without restoring any state, 729 * so we should not arrive here - there is a direct handoff between 730 * the interrupt stub and softint_dispatch(). 731 */ 732 void 733 softint_thread(void *cookie) 734 { 735 736 panic("softint_thread"); 737 } 738 739 /* 740 * softint_dispatch: 741 * 742 * Fast path: entry point from machine-dependent code. 743 */ 744 void 745 softint_dispatch(lwp_t *pinned, int s) 746 { 747 struct bintime now; 748 softint_t *si; 749 u_int timing; 750 lwp_t *l; 751 752 l = curlwp; 753 si = l->l_private; 754 755 /* 756 * Note the interrupted LWP, and mark the current LWP as running 757 * before proceeding. Although this must as a rule be done with 758 * the LWP locked, at this point no external agents will want to 759 * modify the interrupt LWP's state. 760 */ 761 timing = (softint_timing ? LW_TIMEINTR : 0); 762 l->l_switchto = pinned; 763 l->l_stat = LSONPROC; 764 l->l_flag |= (LW_RUNNING | timing); 765 766 /* 767 * Dispatch the interrupt. If softints are being timed, charge 768 * for it. 769 */ 770 if (timing) 771 binuptime(&l->l_stime); 772 softint_execute(si, l, s); 773 if (timing) { 774 binuptime(&now); 775 updatertime(l, &now); 776 l->l_flag &= ~LW_TIMEINTR; 777 } 778 779 /* 780 * If we blocked while handling the interrupt, the pinned LWP is 781 * gone so switch to the idle LWP. It will select a new LWP to 782 * run. 783 * 784 * We must drop the priority level as switching at IPL_HIGH could 785 * deadlock the system. We have already set si->si_active = 0, 786 * which means another interrupt at this level can be triggered. 787 * That's not be a problem: we are lowering to level 's' which will 788 * prevent softint_dispatch() from being reentered at level 's', 789 * until the priority is finally dropped to IPL_NONE on entry to 790 * the idle loop. 791 */ 792 l->l_stat = LSIDL; 793 if (l->l_switchto == NULL) { 794 splx(s); 795 pmap_deactivate(l); 796 lwp_exit_switchaway(l); 797 /* NOTREACHED */ 798 } 799 l->l_switchto = NULL; 800 l->l_flag &= ~LW_RUNNING; 801 } 802 803 #endif /* !__HAVE_FAST_SOFTINTS */ 804