1 /* $NetBSD: kern_synch.c,v 1.280 2010/03/03 00:47:31 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 11 * Daniel Sieger. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1990, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.280 2010/03/03 00:47:31 yamt Exp $"); 73 74 #include "opt_kstack.h" 75 #include "opt_perfctrs.h" 76 #include "opt_sa.h" 77 #include "opt_dtrace.h" 78 79 #define __MUTEX_PRIVATE 80 81 #include <sys/param.h> 82 #include <sys/systm.h> 83 #include <sys/proc.h> 84 #include <sys/kernel.h> 85 #if defined(PERFCTRS) 86 #include <sys/pmc.h> 87 #endif 88 #include <sys/cpu.h> 89 #include <sys/resourcevar.h> 90 #include <sys/sched.h> 91 #include <sys/sa.h> 92 #include <sys/savar.h> 93 #include <sys/syscall_stats.h> 94 #include <sys/sleepq.h> 95 #include <sys/lockdebug.h> 96 #include <sys/evcnt.h> 97 #include <sys/intr.h> 98 #include <sys/lwpctl.h> 99 #include <sys/atomic.h> 100 #include <sys/simplelock.h> 101 102 #include <uvm/uvm_extern.h> 103 104 #include <dev/lockstat.h> 105 106 #include <sys/dtrace_bsd.h> 107 int dtrace_vtime_active=0; 108 dtrace_vtime_switch_func_t dtrace_vtime_switch_func; 109 110 static void sched_unsleep(struct lwp *, bool); 111 static void sched_changepri(struct lwp *, pri_t); 112 static void sched_lendpri(struct lwp *, pri_t); 113 static void resched_cpu(struct lwp *); 114 115 syncobj_t sleep_syncobj = { 116 SOBJ_SLEEPQ_SORTED, 117 sleepq_unsleep, 118 sleepq_changepri, 119 sleepq_lendpri, 120 syncobj_noowner, 121 }; 122 123 syncobj_t sched_syncobj = { 124 SOBJ_SLEEPQ_SORTED, 125 sched_unsleep, 126 sched_changepri, 127 sched_lendpri, 128 syncobj_noowner, 129 }; 130 131 callout_t sched_pstats_ch; 132 unsigned sched_pstats_ticks; 133 kcondvar_t lbolt; /* once a second sleep address */ 134 135 /* Preemption event counters */ 136 static struct evcnt kpreempt_ev_crit; 137 static struct evcnt kpreempt_ev_klock; 138 static struct evcnt kpreempt_ev_immed; 139 140 /* 141 * During autoconfiguration or after a panic, a sleep will simply lower the 142 * priority briefly to allow interrupts, then return. The priority to be 143 * used (safepri) is machine-dependent, thus this value is initialized and 144 * maintained in the machine-dependent layers. This priority will typically 145 * be 0, or the lowest priority that is safe for use on the interrupt stack; 146 * it can be made higher to block network software interrupts after panics. 147 */ 148 int safepri; 149 150 void 151 synch_init(void) 152 { 153 154 cv_init(&lbolt, "lbolt"); 155 callout_init(&sched_pstats_ch, CALLOUT_MPSAFE); 156 callout_setfunc(&sched_pstats_ch, sched_pstats, NULL); 157 158 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL, 159 "kpreempt", "defer: critical section"); 160 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL, 161 "kpreempt", "defer: kernel_lock"); 162 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL, 163 "kpreempt", "immediate"); 164 165 sched_pstats(NULL); 166 } 167 168 /* 169 * OBSOLETE INTERFACE 170 * 171 * General sleep call. Suspends the current LWP until a wakeup is 172 * performed on the specified identifier. The LWP will then be made 173 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 174 * means no timeout). If pri includes PCATCH flag, signals are checked 175 * before and after sleeping, else signals are not checked. Returns 0 if 176 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 177 * signal needs to be delivered, ERESTART is returned if the current system 178 * call should be restarted if possible, and EINTR is returned if the system 179 * call should be interrupted by the signal (return EINTR). 180 * 181 * The interlock is held until we are on a sleep queue. The interlock will 182 * be locked before returning back to the caller unless the PNORELOCK flag 183 * is specified, in which case the interlock will always be unlocked upon 184 * return. 185 */ 186 int 187 ltsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 188 volatile struct simplelock *interlock) 189 { 190 struct lwp *l = curlwp; 191 sleepq_t *sq; 192 kmutex_t *mp; 193 int error; 194 195 KASSERT((l->l_pflag & LP_INTR) == 0); 196 KASSERT(ident != &lbolt); 197 198 if (sleepq_dontsleep(l)) { 199 (void)sleepq_abort(NULL, 0); 200 if ((priority & PNORELOCK) != 0) 201 simple_unlock(interlock); 202 return 0; 203 } 204 205 l->l_kpriority = true; 206 sq = sleeptab_lookup(&sleeptab, ident, &mp); 207 sleepq_enter(sq, l, mp); 208 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 209 210 if (interlock != NULL) { 211 KASSERT(simple_lock_held(interlock)); 212 simple_unlock(interlock); 213 } 214 215 error = sleepq_block(timo, priority & PCATCH); 216 217 if (interlock != NULL && (priority & PNORELOCK) == 0) 218 simple_lock(interlock); 219 220 return error; 221 } 222 223 int 224 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 225 kmutex_t *mtx) 226 { 227 struct lwp *l = curlwp; 228 sleepq_t *sq; 229 kmutex_t *mp; 230 int error; 231 232 KASSERT((l->l_pflag & LP_INTR) == 0); 233 KASSERT(ident != &lbolt); 234 235 if (sleepq_dontsleep(l)) { 236 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 237 return 0; 238 } 239 240 l->l_kpriority = true; 241 sq = sleeptab_lookup(&sleeptab, ident, &mp); 242 sleepq_enter(sq, l, mp); 243 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 244 mutex_exit(mtx); 245 error = sleepq_block(timo, priority & PCATCH); 246 247 if ((priority & PNORELOCK) == 0) 248 mutex_enter(mtx); 249 250 return error; 251 } 252 253 /* 254 * General sleep call for situations where a wake-up is not expected. 255 */ 256 int 257 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 258 { 259 struct lwp *l = curlwp; 260 kmutex_t *mp; 261 sleepq_t *sq; 262 int error; 263 264 if (sleepq_dontsleep(l)) 265 return sleepq_abort(NULL, 0); 266 267 if (mtx != NULL) 268 mutex_exit(mtx); 269 l->l_kpriority = true; 270 sq = sleeptab_lookup(&sleeptab, l, &mp); 271 sleepq_enter(sq, l, mp); 272 sleepq_enqueue(sq, l, wmesg, &sleep_syncobj); 273 error = sleepq_block(timo, intr); 274 if (mtx != NULL) 275 mutex_enter(mtx); 276 277 return error; 278 } 279 280 #ifdef KERN_SA 281 /* 282 * sa_awaken: 283 * 284 * We believe this lwp is an SA lwp. If it's yielding, 285 * let it know it needs to wake up. 286 * 287 * We are called and exit with the lwp locked. We are 288 * called in the middle of wakeup operations, so we need 289 * to not touch the locks at all. 290 */ 291 void 292 sa_awaken(struct lwp *l) 293 { 294 /* LOCK_ASSERT(lwp_locked(l, NULL)); */ 295 296 if (l == l->l_savp->savp_lwp && l->l_flag & LW_SA_YIELD) 297 l->l_flag &= ~LW_SA_IDLE; 298 } 299 #endif /* KERN_SA */ 300 301 /* 302 * OBSOLETE INTERFACE 303 * 304 * Make all LWPs sleeping on the specified identifier runnable. 305 */ 306 void 307 wakeup(wchan_t ident) 308 { 309 sleepq_t *sq; 310 kmutex_t *mp; 311 312 if (__predict_false(cold)) 313 return; 314 315 sq = sleeptab_lookup(&sleeptab, ident, &mp); 316 sleepq_wake(sq, ident, (u_int)-1, mp); 317 } 318 319 /* 320 * OBSOLETE INTERFACE 321 * 322 * Make the highest priority LWP first in line on the specified 323 * identifier runnable. 324 */ 325 void 326 wakeup_one(wchan_t ident) 327 { 328 sleepq_t *sq; 329 kmutex_t *mp; 330 331 if (__predict_false(cold)) 332 return; 333 334 sq = sleeptab_lookup(&sleeptab, ident, &mp); 335 sleepq_wake(sq, ident, 1, mp); 336 } 337 338 339 /* 340 * General yield call. Puts the current LWP back on its run queue and 341 * performs a voluntary context switch. Should only be called when the 342 * current LWP explicitly requests it (eg sched_yield(2)). 343 */ 344 void 345 yield(void) 346 { 347 struct lwp *l = curlwp; 348 349 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 350 lwp_lock(l); 351 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 352 KASSERT(l->l_stat == LSONPROC); 353 l->l_kpriority = false; 354 (void)mi_switch(l); 355 KERNEL_LOCK(l->l_biglocks, l); 356 } 357 358 /* 359 * General preemption call. Puts the current LWP back on its run queue 360 * and performs an involuntary context switch. 361 */ 362 void 363 preempt(void) 364 { 365 struct lwp *l = curlwp; 366 367 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 368 lwp_lock(l); 369 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 370 KASSERT(l->l_stat == LSONPROC); 371 l->l_kpriority = false; 372 l->l_nivcsw++; 373 (void)mi_switch(l); 374 KERNEL_LOCK(l->l_biglocks, l); 375 } 376 377 /* 378 * Handle a request made by another agent to preempt the current LWP 379 * in-kernel. Usually called when l_dopreempt may be non-zero. 380 * 381 * Character addresses for lockstat only. 382 */ 383 static char in_critical_section; 384 static char kernel_lock_held; 385 static char is_softint; 386 static char cpu_kpreempt_enter_fail; 387 388 bool 389 kpreempt(uintptr_t where) 390 { 391 uintptr_t failed; 392 lwp_t *l; 393 int s, dop, lsflag; 394 395 l = curlwp; 396 failed = 0; 397 while ((dop = l->l_dopreempt) != 0) { 398 if (l->l_stat != LSONPROC) { 399 /* 400 * About to block (or die), let it happen. 401 * Doesn't really count as "preemption has 402 * been blocked", since we're going to 403 * context switch. 404 */ 405 l->l_dopreempt = 0; 406 return true; 407 } 408 if (__predict_false((l->l_flag & LW_IDLE) != 0)) { 409 /* Can't preempt idle loop, don't count as failure. */ 410 l->l_dopreempt = 0; 411 return true; 412 } 413 if (__predict_false(l->l_nopreempt != 0)) { 414 /* LWP holds preemption disabled, explicitly. */ 415 if ((dop & DOPREEMPT_COUNTED) == 0) { 416 kpreempt_ev_crit.ev_count++; 417 } 418 failed = (uintptr_t)&in_critical_section; 419 break; 420 } 421 if (__predict_false((l->l_pflag & LP_INTR) != 0)) { 422 /* Can't preempt soft interrupts yet. */ 423 l->l_dopreempt = 0; 424 failed = (uintptr_t)&is_softint; 425 break; 426 } 427 s = splsched(); 428 if (__predict_false(l->l_blcnt != 0 || 429 curcpu()->ci_biglock_wanted != NULL)) { 430 /* Hold or want kernel_lock, code is not MT safe. */ 431 splx(s); 432 if ((dop & DOPREEMPT_COUNTED) == 0) { 433 kpreempt_ev_klock.ev_count++; 434 } 435 failed = (uintptr_t)&kernel_lock_held; 436 break; 437 } 438 if (__predict_false(!cpu_kpreempt_enter(where, s))) { 439 /* 440 * It may be that the IPL is too high. 441 * kpreempt_enter() can schedule an 442 * interrupt to retry later. 443 */ 444 splx(s); 445 failed = (uintptr_t)&cpu_kpreempt_enter_fail; 446 break; 447 } 448 /* Do it! */ 449 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) { 450 kpreempt_ev_immed.ev_count++; 451 } 452 lwp_lock(l); 453 mi_switch(l); 454 l->l_nopreempt++; 455 splx(s); 456 457 /* Take care of any MD cleanup. */ 458 cpu_kpreempt_exit(where); 459 l->l_nopreempt--; 460 } 461 462 if (__predict_true(!failed)) { 463 return false; 464 } 465 466 /* Record preemption failure for reporting via lockstat. */ 467 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED); 468 lsflag = 0; 469 LOCKSTAT_ENTER(lsflag); 470 if (__predict_false(lsflag)) { 471 if (where == 0) { 472 where = (uintptr_t)__builtin_return_address(0); 473 } 474 /* Preemption is on, might recurse, so make it atomic. */ 475 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL, 476 (void *)where) == NULL) { 477 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime); 478 l->l_pfaillock = failed; 479 } 480 } 481 LOCKSTAT_EXIT(lsflag); 482 return true; 483 } 484 485 /* 486 * Return true if preemption is explicitly disabled. 487 */ 488 bool 489 kpreempt_disabled(void) 490 { 491 const lwp_t *l = curlwp; 492 493 return l->l_nopreempt != 0 || l->l_stat == LSZOMB || 494 (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled(); 495 } 496 497 /* 498 * Disable kernel preemption. 499 */ 500 void 501 kpreempt_disable(void) 502 { 503 504 KPREEMPT_DISABLE(curlwp); 505 } 506 507 /* 508 * Reenable kernel preemption. 509 */ 510 void 511 kpreempt_enable(void) 512 { 513 514 KPREEMPT_ENABLE(curlwp); 515 } 516 517 /* 518 * Compute the amount of time during which the current lwp was running. 519 * 520 * - update l_rtime unless it's an idle lwp. 521 */ 522 523 void 524 updatertime(lwp_t *l, const struct bintime *now) 525 { 526 527 if (__predict_false(l->l_flag & LW_IDLE)) 528 return; 529 530 /* rtime += now - stime */ 531 bintime_add(&l->l_rtime, now); 532 bintime_sub(&l->l_rtime, &l->l_stime); 533 } 534 535 /* 536 * Select next LWP from the current CPU to run.. 537 */ 538 static inline lwp_t * 539 nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc) 540 { 541 lwp_t *newl; 542 543 /* 544 * Let sched_nextlwp() select the LWP to run the CPU next. 545 * If no LWP is runnable, select the idle LWP. 546 * 547 * Note that spc_lwplock might not necessary be held, and 548 * new thread would be unlocked after setting the LWP-lock. 549 */ 550 newl = sched_nextlwp(); 551 if (newl != NULL) { 552 sched_dequeue(newl); 553 KASSERT(lwp_locked(newl, spc->spc_mutex)); 554 KASSERT(newl->l_cpu == ci); 555 newl->l_stat = LSONPROC; 556 newl->l_pflag |= LP_RUNNING; 557 lwp_setlock(newl, spc->spc_lwplock); 558 } else { 559 newl = ci->ci_data.cpu_idlelwp; 560 newl->l_stat = LSONPROC; 561 newl->l_pflag |= LP_RUNNING; 562 } 563 564 /* 565 * Only clear want_resched if there are no pending (slow) 566 * software interrupts. 567 */ 568 ci->ci_want_resched = ci->ci_data.cpu_softints; 569 spc->spc_flags &= ~SPCF_SWITCHCLEAR; 570 spc->spc_curpriority = lwp_eprio(newl); 571 572 return newl; 573 } 574 575 /* 576 * The machine independent parts of context switch. 577 * 578 * Returns 1 if another LWP was actually run. 579 */ 580 int 581 mi_switch(lwp_t *l) 582 { 583 struct cpu_info *ci; 584 struct schedstate_percpu *spc; 585 struct lwp *newl; 586 int retval, oldspl; 587 struct bintime bt; 588 bool returning; 589 590 KASSERT(lwp_locked(l, NULL)); 591 KASSERT(kpreempt_disabled()); 592 LOCKDEBUG_BARRIER(l->l_mutex, 1); 593 594 kstack_check_magic(l); 595 596 binuptime(&bt); 597 598 KASSERT((l->l_pflag & LP_RUNNING) != 0); 599 KASSERT(l->l_cpu == curcpu()); 600 ci = l->l_cpu; 601 spc = &ci->ci_schedstate; 602 returning = false; 603 newl = NULL; 604 605 /* 606 * If we have been asked to switch to a specific LWP, then there 607 * is no need to inspect the run queues. If a soft interrupt is 608 * blocking, then return to the interrupted thread without adjusting 609 * VM context or its start time: neither have been changed in order 610 * to take the interrupt. 611 */ 612 if (l->l_switchto != NULL) { 613 if ((l->l_pflag & LP_INTR) != 0) { 614 returning = true; 615 softint_block(l); 616 if ((l->l_pflag & LP_TIMEINTR) != 0) 617 updatertime(l, &bt); 618 } 619 newl = l->l_switchto; 620 l->l_switchto = NULL; 621 } 622 #ifndef __HAVE_FAST_SOFTINTS 623 else if (ci->ci_data.cpu_softints != 0) { 624 /* There are pending soft interrupts, so pick one. */ 625 newl = softint_picklwp(); 626 newl->l_stat = LSONPROC; 627 newl->l_pflag |= LP_RUNNING; 628 } 629 #endif /* !__HAVE_FAST_SOFTINTS */ 630 631 /* Count time spent in current system call */ 632 if (!returning) { 633 SYSCALL_TIME_SLEEP(l); 634 635 /* 636 * XXXSMP If we are using h/w performance counters, 637 * save context. 638 */ 639 #if PERFCTRS 640 if (PMC_ENABLED(l->l_proc)) { 641 pmc_save_context(l->l_proc); 642 } 643 #endif 644 updatertime(l, &bt); 645 } 646 647 /* Lock the runqueue */ 648 KASSERT(l->l_stat != LSRUN); 649 mutex_spin_enter(spc->spc_mutex); 650 651 /* 652 * If on the CPU and we have gotten this far, then we must yield. 653 */ 654 if (l->l_stat == LSONPROC && l != newl) { 655 KASSERT(lwp_locked(l, spc->spc_lwplock)); 656 if ((l->l_flag & LW_IDLE) == 0) { 657 l->l_stat = LSRUN; 658 lwp_setlock(l, spc->spc_mutex); 659 sched_enqueue(l, true); 660 /* Handle migration case */ 661 KASSERT(spc->spc_migrating == NULL); 662 if (l->l_target_cpu != NULL) { 663 spc->spc_migrating = l; 664 } 665 } else 666 l->l_stat = LSIDL; 667 } 668 669 /* Pick new LWP to run. */ 670 if (newl == NULL) { 671 newl = nextlwp(ci, spc); 672 } 673 674 /* Items that must be updated with the CPU locked. */ 675 if (!returning) { 676 /* Update the new LWP's start time. */ 677 newl->l_stime = bt; 678 679 /* 680 * ci_curlwp changes when a fast soft interrupt occurs. 681 * We use cpu_onproc to keep track of which kernel or 682 * user thread is running 'underneath' the software 683 * interrupt. This is important for time accounting, 684 * itimers and forcing user threads to preempt (aston). 685 */ 686 ci->ci_data.cpu_onproc = newl; 687 } 688 689 /* 690 * Preemption related tasks. Must be done with the current 691 * CPU locked. 692 */ 693 cpu_did_resched(l); 694 l->l_dopreempt = 0; 695 if (__predict_false(l->l_pfailaddr != 0)) { 696 LOCKSTAT_FLAG(lsflag); 697 LOCKSTAT_ENTER(lsflag); 698 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime); 699 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN, 700 1, l->l_pfailtime, l->l_pfailaddr); 701 LOCKSTAT_EXIT(lsflag); 702 l->l_pfailtime = 0; 703 l->l_pfaillock = 0; 704 l->l_pfailaddr = 0; 705 } 706 707 if (l != newl) { 708 struct lwp *prevlwp; 709 710 /* Release all locks, but leave the current LWP locked */ 711 if (l->l_mutex == spc->spc_mutex) { 712 /* 713 * Drop spc_lwplock, if the current LWP has been moved 714 * to the run queue (it is now locked by spc_mutex). 715 */ 716 mutex_spin_exit(spc->spc_lwplock); 717 } else { 718 /* 719 * Otherwise, drop the spc_mutex, we are done with the 720 * run queues. 721 */ 722 mutex_spin_exit(spc->spc_mutex); 723 } 724 725 /* 726 * Mark that context switch is going to be performed 727 * for this LWP, to protect it from being switched 728 * to on another CPU. 729 */ 730 KASSERT(l->l_ctxswtch == 0); 731 l->l_ctxswtch = 1; 732 l->l_ncsw++; 733 KASSERT((l->l_pflag & LP_RUNNING) != 0); 734 l->l_pflag &= ~LP_RUNNING; 735 736 /* 737 * Increase the count of spin-mutexes before the release 738 * of the last lock - we must remain at IPL_SCHED during 739 * the context switch. 740 */ 741 oldspl = MUTEX_SPIN_OLDSPL(ci); 742 ci->ci_mtx_count--; 743 lwp_unlock(l); 744 745 /* Count the context switch on this CPU. */ 746 ci->ci_data.cpu_nswtch++; 747 748 /* Update status for lwpctl, if present. */ 749 if (l->l_lwpctl != NULL) 750 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE; 751 752 /* 753 * Save old VM context, unless a soft interrupt 754 * handler is blocking. 755 */ 756 if (!returning) 757 pmap_deactivate(l); 758 759 /* 760 * We may need to spin-wait if 'newl' is still 761 * context switching on another CPU. 762 */ 763 if (__predict_false(newl->l_ctxswtch != 0)) { 764 u_int count; 765 count = SPINLOCK_BACKOFF_MIN; 766 while (newl->l_ctxswtch) 767 SPINLOCK_BACKOFF(count); 768 } 769 770 /* 771 * If DTrace has set the active vtime enum to anything 772 * other than INACTIVE (0), then it should have set the 773 * function to call. 774 */ 775 if (__predict_false(dtrace_vtime_active)) { 776 (*dtrace_vtime_switch_func)(newl); 777 } 778 779 /* Switch to the new LWP.. */ 780 prevlwp = cpu_switchto(l, newl, returning); 781 ci = curcpu(); 782 783 /* 784 * Switched away - we have new curlwp. 785 * Restore VM context and IPL. 786 */ 787 pmap_activate(l); 788 uvm_emap_switch(l); 789 790 if (prevlwp != NULL) { 791 /* Normalize the count of the spin-mutexes */ 792 ci->ci_mtx_count++; 793 /* Unmark the state of context switch */ 794 membar_exit(); 795 prevlwp->l_ctxswtch = 0; 796 } 797 798 /* Update status for lwpctl, if present. */ 799 if (l->l_lwpctl != NULL) { 800 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); 801 l->l_lwpctl->lc_pctr++; 802 } 803 804 KASSERT(l->l_cpu == ci); 805 splx(oldspl); 806 retval = 1; 807 } else { 808 /* Nothing to do - just unlock and return. */ 809 mutex_spin_exit(spc->spc_mutex); 810 lwp_unlock(l); 811 retval = 0; 812 } 813 814 KASSERT(l == curlwp); 815 KASSERT(l->l_stat == LSONPROC); 816 817 /* 818 * XXXSMP If we are using h/w performance counters, restore context. 819 * XXXSMP preemption problem. 820 */ 821 #if PERFCTRS 822 if (PMC_ENABLED(l->l_proc)) { 823 pmc_restore_context(l->l_proc); 824 } 825 #endif 826 SYSCALL_TIME_WAKEUP(l); 827 LOCKDEBUG_BARRIER(NULL, 1); 828 829 return retval; 830 } 831 832 /* 833 * The machine independent parts of context switch to oblivion. 834 * Does not return. Call with the LWP unlocked. 835 */ 836 void 837 lwp_exit_switchaway(lwp_t *l) 838 { 839 struct cpu_info *ci; 840 struct lwp *newl; 841 struct bintime bt; 842 843 ci = l->l_cpu; 844 845 KASSERT(kpreempt_disabled()); 846 KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL); 847 KASSERT(ci == curcpu()); 848 LOCKDEBUG_BARRIER(NULL, 0); 849 850 kstack_check_magic(l); 851 852 /* Count time spent in current system call */ 853 SYSCALL_TIME_SLEEP(l); 854 binuptime(&bt); 855 updatertime(l, &bt); 856 857 /* Must stay at IPL_SCHED even after releasing run queue lock. */ 858 (void)splsched(); 859 860 /* 861 * Let sched_nextlwp() select the LWP to run the CPU next. 862 * If no LWP is runnable, select the idle LWP. 863 * 864 * Note that spc_lwplock might not necessary be held, and 865 * new thread would be unlocked after setting the LWP-lock. 866 */ 867 spc_lock(ci); 868 #ifndef __HAVE_FAST_SOFTINTS 869 if (ci->ci_data.cpu_softints != 0) { 870 /* There are pending soft interrupts, so pick one. */ 871 newl = softint_picklwp(); 872 newl->l_stat = LSONPROC; 873 newl->l_pflag |= LP_RUNNING; 874 } else 875 #endif /* !__HAVE_FAST_SOFTINTS */ 876 { 877 newl = nextlwp(ci, &ci->ci_schedstate); 878 } 879 880 /* Update the new LWP's start time. */ 881 newl->l_stime = bt; 882 l->l_pflag &= ~LP_RUNNING; 883 884 /* 885 * ci_curlwp changes when a fast soft interrupt occurs. 886 * We use cpu_onproc to keep track of which kernel or 887 * user thread is running 'underneath' the software 888 * interrupt. This is important for time accounting, 889 * itimers and forcing user threads to preempt (aston). 890 */ 891 ci->ci_data.cpu_onproc = newl; 892 893 /* 894 * Preemption related tasks. Must be done with the current 895 * CPU locked. 896 */ 897 cpu_did_resched(l); 898 899 /* Unlock the run queue. */ 900 spc_unlock(ci); 901 902 /* Count the context switch on this CPU. */ 903 ci->ci_data.cpu_nswtch++; 904 905 /* Update status for lwpctl, if present. */ 906 if (l->l_lwpctl != NULL) 907 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 908 909 /* 910 * We may need to spin-wait if 'newl' is still 911 * context switching on another CPU. 912 */ 913 if (__predict_false(newl->l_ctxswtch != 0)) { 914 u_int count; 915 count = SPINLOCK_BACKOFF_MIN; 916 while (newl->l_ctxswtch) 917 SPINLOCK_BACKOFF(count); 918 } 919 920 /* 921 * If DTrace has set the active vtime enum to anything 922 * other than INACTIVE (0), then it should have set the 923 * function to call. 924 */ 925 if (__predict_false(dtrace_vtime_active)) { 926 (*dtrace_vtime_switch_func)(newl); 927 } 928 929 /* Switch to the new LWP.. */ 930 (void)cpu_switchto(NULL, newl, false); 931 932 for (;;) continue; /* XXX: convince gcc about "noreturn" */ 933 /* NOTREACHED */ 934 } 935 936 /* 937 * setrunnable: change LWP state to be runnable, placing it on the run queue. 938 * 939 * Call with the process and LWP locked. Will return with the LWP unlocked. 940 */ 941 void 942 setrunnable(struct lwp *l) 943 { 944 struct proc *p = l->l_proc; 945 struct cpu_info *ci; 946 947 KASSERT((l->l_flag & LW_IDLE) == 0); 948 KASSERT(mutex_owned(p->p_lock)); 949 KASSERT(lwp_locked(l, NULL)); 950 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex); 951 952 switch (l->l_stat) { 953 case LSSTOP: 954 /* 955 * If we're being traced (possibly because someone attached us 956 * while we were stopped), check for a signal from the debugger. 957 */ 958 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0) 959 signotify(l); 960 p->p_nrlwps++; 961 break; 962 case LSSUSPENDED: 963 l->l_flag &= ~LW_WSUSPEND; 964 p->p_nrlwps++; 965 cv_broadcast(&p->p_lwpcv); 966 break; 967 case LSSLEEP: 968 KASSERT(l->l_wchan != NULL); 969 break; 970 default: 971 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 972 } 973 974 #ifdef KERN_SA 975 if (l->l_proc->p_sa) 976 sa_awaken(l); 977 #endif /* KERN_SA */ 978 979 /* 980 * If the LWP was sleeping interruptably, then it's OK to start it 981 * again. If not, mark it as still sleeping. 982 */ 983 if (l->l_wchan != NULL) { 984 l->l_stat = LSSLEEP; 985 /* lwp_unsleep() will release the lock. */ 986 lwp_unsleep(l, true); 987 return; 988 } 989 990 /* 991 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 992 * about to call mi_switch(), in which case it will yield. 993 */ 994 if ((l->l_pflag & LP_RUNNING) != 0) { 995 l->l_stat = LSONPROC; 996 l->l_slptime = 0; 997 lwp_unlock(l); 998 return; 999 } 1000 1001 /* 1002 * Look for a CPU to run. 1003 * Set the LWP runnable. 1004 */ 1005 ci = sched_takecpu(l); 1006 l->l_cpu = ci; 1007 spc_lock(ci); 1008 lwp_unlock_to(l, ci->ci_schedstate.spc_mutex); 1009 sched_setrunnable(l); 1010 l->l_stat = LSRUN; 1011 l->l_slptime = 0; 1012 1013 sched_enqueue(l, false); 1014 resched_cpu(l); 1015 lwp_unlock(l); 1016 } 1017 1018 /* 1019 * suspendsched: 1020 * 1021 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 1022 */ 1023 void 1024 suspendsched(void) 1025 { 1026 CPU_INFO_ITERATOR cii; 1027 struct cpu_info *ci; 1028 struct lwp *l; 1029 struct proc *p; 1030 1031 /* 1032 * We do this by process in order not to violate the locking rules. 1033 */ 1034 mutex_enter(proc_lock); 1035 PROCLIST_FOREACH(p, &allproc) { 1036 mutex_enter(p->p_lock); 1037 if ((p->p_flag & PK_SYSTEM) != 0) { 1038 mutex_exit(p->p_lock); 1039 continue; 1040 } 1041 1042 p->p_stat = SSTOP; 1043 1044 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1045 if (l == curlwp) 1046 continue; 1047 1048 lwp_lock(l); 1049 1050 /* 1051 * Set L_WREBOOT so that the LWP will suspend itself 1052 * when it tries to return to user mode. We want to 1053 * try and get to get as many LWPs as possible to 1054 * the user / kernel boundary, so that they will 1055 * release any locks that they hold. 1056 */ 1057 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 1058 1059 if (l->l_stat == LSSLEEP && 1060 (l->l_flag & LW_SINTR) != 0) { 1061 /* setrunnable() will release the lock. */ 1062 setrunnable(l); 1063 continue; 1064 } 1065 1066 lwp_unlock(l); 1067 } 1068 1069 mutex_exit(p->p_lock); 1070 } 1071 mutex_exit(proc_lock); 1072 1073 /* 1074 * Kick all CPUs to make them preempt any LWPs running in user mode. 1075 * They'll trap into the kernel and suspend themselves in userret(). 1076 */ 1077 for (CPU_INFO_FOREACH(cii, ci)) { 1078 spc_lock(ci); 1079 cpu_need_resched(ci, RESCHED_IMMED); 1080 spc_unlock(ci); 1081 } 1082 } 1083 1084 /* 1085 * sched_unsleep: 1086 * 1087 * The is called when the LWP has not been awoken normally but instead 1088 * interrupted: for example, if the sleep timed out. Because of this, 1089 * it's not a valid action for running or idle LWPs. 1090 */ 1091 static void 1092 sched_unsleep(struct lwp *l, bool cleanup) 1093 { 1094 1095 lwp_unlock(l); 1096 panic("sched_unsleep"); 1097 } 1098 1099 static void 1100 resched_cpu(struct lwp *l) 1101 { 1102 struct cpu_info *ci = l->l_cpu; 1103 1104 KASSERT(lwp_locked(l, NULL)); 1105 if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority) 1106 cpu_need_resched(ci, 0); 1107 } 1108 1109 static void 1110 sched_changepri(struct lwp *l, pri_t pri) 1111 { 1112 1113 KASSERT(lwp_locked(l, NULL)); 1114 1115 if (l->l_stat == LSRUN) { 1116 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1117 sched_dequeue(l); 1118 l->l_priority = pri; 1119 sched_enqueue(l, false); 1120 } else { 1121 l->l_priority = pri; 1122 } 1123 resched_cpu(l); 1124 } 1125 1126 static void 1127 sched_lendpri(struct lwp *l, pri_t pri) 1128 { 1129 1130 KASSERT(lwp_locked(l, NULL)); 1131 1132 if (l->l_stat == LSRUN) { 1133 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1134 sched_dequeue(l); 1135 l->l_inheritedprio = pri; 1136 sched_enqueue(l, false); 1137 } else { 1138 l->l_inheritedprio = pri; 1139 } 1140 resched_cpu(l); 1141 } 1142 1143 struct lwp * 1144 syncobj_noowner(wchan_t wchan) 1145 { 1146 1147 return NULL; 1148 } 1149 1150 /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */ 1151 const fixpt_t ccpu = 0.95122942450071400909 * FSCALE; 1152 1153 /* 1154 * sched_pstats: 1155 * 1156 * Update process statistics and check CPU resource allocation. 1157 * Call scheduler-specific hook to eventually adjust process/LWP 1158 * priorities. 1159 */ 1160 void 1161 sched_pstats(void *arg) 1162 { 1163 const int clkhz = (stathz != 0 ? stathz : hz); 1164 static bool backwards; 1165 struct rlimit *rlim; 1166 struct lwp *l; 1167 struct proc *p; 1168 long runtm; 1169 fixpt_t lpctcpu; 1170 u_int lcpticks; 1171 int sig; 1172 1173 sched_pstats_ticks++; 1174 1175 mutex_enter(proc_lock); 1176 PROCLIST_FOREACH(p, &allproc) { 1177 /* Increment sleep time (if sleeping), ignore overflow. */ 1178 mutex_enter(p->p_lock); 1179 runtm = p->p_rtime.sec; 1180 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1181 if (__predict_false((l->l_flag & LW_IDLE) != 0)) 1182 continue; 1183 lwp_lock(l); 1184 runtm += l->l_rtime.sec; 1185 l->l_swtime++; 1186 sched_lwp_stats(l); 1187 lwp_unlock(l); 1188 1189 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 1190 if (l->l_slptime != 0) 1191 continue; 1192 1193 lpctcpu = l->l_pctcpu; 1194 lcpticks = atomic_swap_uint(&l->l_cpticks, 0); 1195 lpctcpu += ((FSCALE - ccpu) * 1196 (lcpticks * FSCALE / clkhz)) >> FSHIFT; 1197 l->l_pctcpu = lpctcpu; 1198 } 1199 /* Calculating p_pctcpu only for ps(1) */ 1200 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 1201 1202 /* 1203 * Check if the process exceeds its CPU resource allocation. 1204 * If over max, kill it. 1205 */ 1206 rlim = &p->p_rlimit[RLIMIT_CPU]; 1207 sig = 0; 1208 if (__predict_false(runtm >= rlim->rlim_cur)) { 1209 if (runtm >= rlim->rlim_max) 1210 sig = SIGKILL; 1211 else { 1212 sig = SIGXCPU; 1213 if (rlim->rlim_cur < rlim->rlim_max) 1214 rlim->rlim_cur += 5; 1215 } 1216 } 1217 mutex_exit(p->p_lock); 1218 if (__predict_false(runtm < 0)) { 1219 if (!backwards) { 1220 backwards = true; 1221 printf("WARNING: negative runtime; " 1222 "monotonic clock has gone backwards\n"); 1223 } 1224 } else if (__predict_false(sig)) { 1225 KASSERT((p->p_flag & PK_SYSTEM) == 0); 1226 psignal(p, sig); 1227 } 1228 } 1229 mutex_exit(proc_lock); 1230 uvm_meter(); 1231 cv_broadcast(&lbolt); 1232 callout_schedule(&sched_pstats_ch, hz); 1233 } 1234