1 /* $NetBSD: kern_synch.c,v 1.342 2020/02/23 16:27:09 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 11 * Daniel Sieger. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1990, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.342 2020/02/23 16:27:09 ad Exp $"); 73 74 #include "opt_kstack.h" 75 #include "opt_dtrace.h" 76 77 #define __MUTEX_PRIVATE 78 79 #include <sys/param.h> 80 #include <sys/systm.h> 81 #include <sys/proc.h> 82 #include <sys/kernel.h> 83 #include <sys/cpu.h> 84 #include <sys/pserialize.h> 85 #include <sys/resourcevar.h> 86 #include <sys/rwlock.h> 87 #include <sys/sched.h> 88 #include <sys/syscall_stats.h> 89 #include <sys/sleepq.h> 90 #include <sys/lockdebug.h> 91 #include <sys/evcnt.h> 92 #include <sys/intr.h> 93 #include <sys/lwpctl.h> 94 #include <sys/atomic.h> 95 #include <sys/syslog.h> 96 97 #include <uvm/uvm_extern.h> 98 99 #include <dev/lockstat.h> 100 101 #include <sys/dtrace_bsd.h> 102 int dtrace_vtime_active=0; 103 dtrace_vtime_switch_func_t dtrace_vtime_switch_func; 104 105 static void sched_unsleep(struct lwp *, bool); 106 static void sched_changepri(struct lwp *, pri_t); 107 static void sched_lendpri(struct lwp *, pri_t); 108 109 syncobj_t sleep_syncobj = { 110 .sobj_flag = SOBJ_SLEEPQ_SORTED, 111 .sobj_unsleep = sleepq_unsleep, 112 .sobj_changepri = sleepq_changepri, 113 .sobj_lendpri = sleepq_lendpri, 114 .sobj_owner = syncobj_noowner, 115 }; 116 117 syncobj_t sched_syncobj = { 118 .sobj_flag = SOBJ_SLEEPQ_SORTED, 119 .sobj_unsleep = sched_unsleep, 120 .sobj_changepri = sched_changepri, 121 .sobj_lendpri = sched_lendpri, 122 .sobj_owner = syncobj_noowner, 123 }; 124 125 syncobj_t kpause_syncobj = { 126 .sobj_flag = SOBJ_SLEEPQ_NULL, 127 .sobj_unsleep = sleepq_unsleep, 128 .sobj_changepri = sleepq_changepri, 129 .sobj_lendpri = sleepq_lendpri, 130 .sobj_owner = syncobj_noowner, 131 }; 132 133 /* "Lightning bolt": once a second sleep address. */ 134 kcondvar_t lbolt __cacheline_aligned; 135 136 u_int sched_pstats_ticks __cacheline_aligned; 137 138 /* Preemption event counters. */ 139 static struct evcnt kpreempt_ev_crit __cacheline_aligned; 140 static struct evcnt kpreempt_ev_klock __cacheline_aligned; 141 static struct evcnt kpreempt_ev_immed __cacheline_aligned; 142 143 void 144 synch_init(void) 145 { 146 147 cv_init(&lbolt, "lbolt"); 148 149 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL, 150 "kpreempt", "defer: critical section"); 151 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL, 152 "kpreempt", "defer: kernel_lock"); 153 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL, 154 "kpreempt", "immediate"); 155 } 156 157 /* 158 * OBSOLETE INTERFACE 159 * 160 * General sleep call. Suspends the current LWP until a wakeup is 161 * performed on the specified identifier. The LWP will then be made 162 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 163 * means no timeout). If pri includes PCATCH flag, signals are checked 164 * before and after sleeping, else signals are not checked. Returns 0 if 165 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 166 * signal needs to be delivered, ERESTART is returned if the current system 167 * call should be restarted if possible, and EINTR is returned if the system 168 * call should be interrupted by the signal (return EINTR). 169 */ 170 int 171 tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo) 172 { 173 struct lwp *l = curlwp; 174 sleepq_t *sq; 175 kmutex_t *mp; 176 177 KASSERT((l->l_pflag & LP_INTR) == 0); 178 KASSERT(ident != &lbolt); 179 180 if (sleepq_dontsleep(l)) { 181 (void)sleepq_abort(NULL, 0); 182 return 0; 183 } 184 185 l->l_kpriority = true; 186 sq = sleeptab_lookup(&sleeptab, ident, &mp); 187 sleepq_enter(sq, l, mp); 188 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 189 return sleepq_block(timo, priority & PCATCH); 190 } 191 192 int 193 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 194 kmutex_t *mtx) 195 { 196 struct lwp *l = curlwp; 197 sleepq_t *sq; 198 kmutex_t *mp; 199 int error; 200 201 KASSERT((l->l_pflag & LP_INTR) == 0); 202 KASSERT(ident != &lbolt); 203 204 if (sleepq_dontsleep(l)) { 205 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 206 return 0; 207 } 208 209 l->l_kpriority = true; 210 sq = sleeptab_lookup(&sleeptab, ident, &mp); 211 sleepq_enter(sq, l, mp); 212 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 213 mutex_exit(mtx); 214 error = sleepq_block(timo, priority & PCATCH); 215 216 if ((priority & PNORELOCK) == 0) 217 mutex_enter(mtx); 218 219 return error; 220 } 221 222 /* 223 * XXXAD Temporary - for use of UVM only. PLEASE DO NOT USE ELSEWHERE. 224 * Will go once there is a better solution, eg waits interlocked by 225 * pg->interlock. To wake an LWP sleeping with this, you need to hold a 226 * write lock. 227 */ 228 int 229 rwtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 230 krwlock_t *rw) 231 { 232 struct lwp *l = curlwp; 233 sleepq_t *sq; 234 kmutex_t *mp; 235 int error; 236 krw_t op; 237 238 KASSERT((l->l_pflag & LP_INTR) == 0); 239 KASSERT(ident != &lbolt); 240 241 if (sleepq_dontsleep(l)) { 242 (void)sleepq_abort(NULL, (priority & PNORELOCK) != 0); 243 if ((priority & PNORELOCK) != 0) 244 rw_exit(rw); 245 return 0; 246 } 247 248 l->l_kpriority = true; 249 sq = sleeptab_lookup(&sleeptab, ident, &mp); 250 sleepq_enter(sq, l, mp); 251 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 252 op = rw_lock_op(rw); 253 rw_exit(rw); 254 error = sleepq_block(timo, priority & PCATCH); 255 256 if ((priority & PNORELOCK) == 0) 257 rw_enter(rw, op); 258 259 return error; 260 } 261 262 /* 263 * General sleep call for situations where a wake-up is not expected. 264 */ 265 int 266 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 267 { 268 struct lwp *l = curlwp; 269 int error; 270 271 KASSERT(!(timo == 0 && intr == false)); 272 273 if (sleepq_dontsleep(l)) 274 return sleepq_abort(NULL, 0); 275 276 if (mtx != NULL) 277 mutex_exit(mtx); 278 l->l_kpriority = true; 279 lwp_lock(l); 280 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); 281 sleepq_enqueue(NULL, l, wmesg, &kpause_syncobj); 282 error = sleepq_block(timo, intr); 283 if (mtx != NULL) 284 mutex_enter(mtx); 285 286 return error; 287 } 288 289 /* 290 * OBSOLETE INTERFACE 291 * 292 * Make all LWPs sleeping on the specified identifier runnable. 293 */ 294 void 295 wakeup(wchan_t ident) 296 { 297 sleepq_t *sq; 298 kmutex_t *mp; 299 300 if (__predict_false(cold)) 301 return; 302 303 sq = sleeptab_lookup(&sleeptab, ident, &mp); 304 sleepq_wake(sq, ident, (u_int)-1, mp); 305 } 306 307 /* 308 * General yield call. Puts the current LWP back on its run queue and 309 * performs a voluntary context switch. Should only be called when the 310 * current LWP explicitly requests it (eg sched_yield(2)). 311 */ 312 void 313 yield(void) 314 { 315 struct lwp *l = curlwp; 316 317 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 318 lwp_lock(l); 319 320 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 321 KASSERT(l->l_stat == LSONPROC); 322 323 /* Voluntary - ditch kpriority boost. */ 324 l->l_kpriority = false; 325 spc_lock(l->l_cpu); 326 mi_switch(l); 327 KERNEL_LOCK(l->l_biglocks, l); 328 } 329 330 /* 331 * General preemption call. Puts the current LWP back on its run queue 332 * and performs an involuntary context switch. 333 */ 334 void 335 preempt(void) 336 { 337 struct lwp *l = curlwp; 338 339 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 340 lwp_lock(l); 341 342 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 343 KASSERT(l->l_stat == LSONPROC); 344 345 /* Involuntary - keep kpriority boost. */ 346 l->l_pflag |= LP_PREEMPTING; 347 spc_lock(l->l_cpu); 348 mi_switch(l); 349 KERNEL_LOCK(l->l_biglocks, l); 350 } 351 352 /* 353 * Handle a request made by another agent to preempt the current LWP 354 * in-kernel. Usually called when l_dopreempt may be non-zero. 355 * 356 * Character addresses for lockstat only. 357 */ 358 static char kpreempt_is_disabled; 359 static char kernel_lock_held; 360 static char is_softint_lwp; 361 static char spl_is_raised; 362 363 bool 364 kpreempt(uintptr_t where) 365 { 366 uintptr_t failed; 367 lwp_t *l; 368 int s, dop, lsflag; 369 370 l = curlwp; 371 failed = 0; 372 while ((dop = l->l_dopreempt) != 0) { 373 if (l->l_stat != LSONPROC) { 374 /* 375 * About to block (or die), let it happen. 376 * Doesn't really count as "preemption has 377 * been blocked", since we're going to 378 * context switch. 379 */ 380 atomic_swap_uint(&l->l_dopreempt, 0); 381 return true; 382 } 383 if (__predict_false((l->l_flag & LW_IDLE) != 0)) { 384 /* Can't preempt idle loop, don't count as failure. */ 385 atomic_swap_uint(&l->l_dopreempt, 0); 386 return true; 387 } 388 if (__predict_false(l->l_nopreempt != 0)) { 389 /* LWP holds preemption disabled, explicitly. */ 390 if ((dop & DOPREEMPT_COUNTED) == 0) { 391 kpreempt_ev_crit.ev_count++; 392 } 393 failed = (uintptr_t)&kpreempt_is_disabled; 394 break; 395 } 396 if (__predict_false((l->l_pflag & LP_INTR) != 0)) { 397 /* Can't preempt soft interrupts yet. */ 398 atomic_swap_uint(&l->l_dopreempt, 0); 399 failed = (uintptr_t)&is_softint_lwp; 400 break; 401 } 402 s = splsched(); 403 if (__predict_false(l->l_blcnt != 0 || 404 curcpu()->ci_biglock_wanted != NULL)) { 405 /* Hold or want kernel_lock, code is not MT safe. */ 406 splx(s); 407 if ((dop & DOPREEMPT_COUNTED) == 0) { 408 kpreempt_ev_klock.ev_count++; 409 } 410 failed = (uintptr_t)&kernel_lock_held; 411 break; 412 } 413 if (__predict_false(!cpu_kpreempt_enter(where, s))) { 414 /* 415 * It may be that the IPL is too high. 416 * kpreempt_enter() can schedule an 417 * interrupt to retry later. 418 */ 419 splx(s); 420 failed = (uintptr_t)&spl_is_raised; 421 break; 422 } 423 /* Do it! */ 424 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) { 425 kpreempt_ev_immed.ev_count++; 426 } 427 lwp_lock(l); 428 /* Involuntary - keep kpriority boost. */ 429 l->l_pflag |= LP_PREEMPTING; 430 spc_lock(l->l_cpu); 431 mi_switch(l); 432 l->l_nopreempt++; 433 splx(s); 434 435 /* Take care of any MD cleanup. */ 436 cpu_kpreempt_exit(where); 437 l->l_nopreempt--; 438 } 439 440 if (__predict_true(!failed)) { 441 return false; 442 } 443 444 /* Record preemption failure for reporting via lockstat. */ 445 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED); 446 lsflag = 0; 447 LOCKSTAT_ENTER(lsflag); 448 if (__predict_false(lsflag)) { 449 if (where == 0) { 450 where = (uintptr_t)__builtin_return_address(0); 451 } 452 /* Preemption is on, might recurse, so make it atomic. */ 453 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL, 454 (void *)where) == NULL) { 455 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime); 456 l->l_pfaillock = failed; 457 } 458 } 459 LOCKSTAT_EXIT(lsflag); 460 return true; 461 } 462 463 /* 464 * Return true if preemption is explicitly disabled. 465 */ 466 bool 467 kpreempt_disabled(void) 468 { 469 const lwp_t *l = curlwp; 470 471 return l->l_nopreempt != 0 || l->l_stat == LSZOMB || 472 (l->l_flag & LW_IDLE) != 0 || (l->l_pflag & LP_INTR) != 0 || 473 cpu_kpreempt_disabled(); 474 } 475 476 /* 477 * Disable kernel preemption. 478 */ 479 void 480 kpreempt_disable(void) 481 { 482 483 KPREEMPT_DISABLE(curlwp); 484 } 485 486 /* 487 * Reenable kernel preemption. 488 */ 489 void 490 kpreempt_enable(void) 491 { 492 493 KPREEMPT_ENABLE(curlwp); 494 } 495 496 /* 497 * Compute the amount of time during which the current lwp was running. 498 * 499 * - update l_rtime unless it's an idle lwp. 500 */ 501 502 void 503 updatertime(lwp_t *l, const struct bintime *now) 504 { 505 506 if (__predict_false(l->l_flag & LW_IDLE)) 507 return; 508 509 /* rtime += now - stime */ 510 bintime_add(&l->l_rtime, now); 511 bintime_sub(&l->l_rtime, &l->l_stime); 512 } 513 514 /* 515 * Select next LWP from the current CPU to run.. 516 */ 517 static inline lwp_t * 518 nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc) 519 { 520 lwp_t *newl; 521 522 /* 523 * Let sched_nextlwp() select the LWP to run the CPU next. 524 * If no LWP is runnable, select the idle LWP. 525 * 526 * On arrival here LWPs on a run queue are locked by spc_mutex which 527 * is currently held. Idle LWPs are always locked by spc_lwplock, 528 * which may or may not be held here. On exit from this code block, 529 * in all cases newl is locked by spc_lwplock. 530 */ 531 newl = sched_nextlwp(); 532 if (newl != NULL) { 533 sched_dequeue(newl); 534 KASSERT(lwp_locked(newl, spc->spc_mutex)); 535 KASSERT(newl->l_cpu == ci); 536 newl->l_stat = LSONPROC; 537 newl->l_pflag |= LP_RUNNING; 538 spc->spc_curpriority = lwp_eprio(newl); 539 spc->spc_flags &= ~(SPCF_SWITCHCLEAR | SPCF_IDLE); 540 lwp_setlock(newl, spc->spc_lwplock); 541 } else { 542 /* 543 * Updates to newl here are unlocked, but newl is the idle 544 * LWP and thus sheltered from outside interference, so no 545 * harm is going to come of it. 546 */ 547 newl = ci->ci_data.cpu_idlelwp; 548 newl->l_stat = LSONPROC; 549 newl->l_pflag |= LP_RUNNING; 550 spc->spc_curpriority = PRI_IDLE; 551 spc->spc_flags = (spc->spc_flags & ~SPCF_SWITCHCLEAR) | 552 SPCF_IDLE; 553 } 554 555 /* 556 * Only clear want_resched if there are no pending (slow) software 557 * interrupts. We can do this without an atomic, because no new 558 * LWPs can appear in the queue due to our hold on spc_mutex, and 559 * the update to ci_want_resched will become globally visible before 560 * the release of spc_mutex becomes globally visible. 561 */ 562 ci->ci_want_resched = ci->ci_data.cpu_softints; 563 564 return newl; 565 } 566 567 /* 568 * The machine independent parts of context switch. 569 * 570 * NOTE: l->l_cpu is not changed in this routine, because an LWP never 571 * changes its own l_cpu (that would screw up curcpu on many ports and could 572 * cause all kinds of other evil stuff). l_cpu is always changed by some 573 * other actor, when it's known the LWP is not running (the LP_RUNNING flag 574 * is checked under lock). 575 */ 576 void 577 mi_switch(lwp_t *l) 578 { 579 struct cpu_info *ci; 580 struct schedstate_percpu *spc; 581 struct lwp *newl; 582 kmutex_t *lock; 583 int oldspl; 584 struct bintime bt; 585 bool returning; 586 587 KASSERT(lwp_locked(l, NULL)); 588 KASSERT(kpreempt_disabled()); 589 KASSERT(mutex_owned(curcpu()->ci_schedstate.spc_mutex)); 590 KASSERTMSG(l->l_blcnt == 0, "kernel_lock leaked"); 591 592 kstack_check_magic(l); 593 594 binuptime(&bt); 595 596 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 597 KASSERT((l->l_pflag & LP_RUNNING) != 0); 598 KASSERT(l->l_cpu == curcpu() || l->l_stat == LSRUN); 599 ci = curcpu(); 600 spc = &ci->ci_schedstate; 601 returning = false; 602 newl = NULL; 603 604 /* 605 * If we have been asked to switch to a specific LWP, then there 606 * is no need to inspect the run queues. If a soft interrupt is 607 * blocking, then return to the interrupted thread without adjusting 608 * VM context or its start time: neither have been changed in order 609 * to take the interrupt. 610 */ 611 if (l->l_switchto != NULL) { 612 if ((l->l_pflag & LP_INTR) != 0) { 613 returning = true; 614 softint_block(l); 615 if ((l->l_pflag & LP_TIMEINTR) != 0) 616 updatertime(l, &bt); 617 } 618 newl = l->l_switchto; 619 l->l_switchto = NULL; 620 } 621 #ifndef __HAVE_FAST_SOFTINTS 622 else if (ci->ci_data.cpu_softints != 0) { 623 /* There are pending soft interrupts, so pick one. */ 624 newl = softint_picklwp(); 625 newl->l_stat = LSONPROC; 626 newl->l_pflag |= LP_RUNNING; 627 } 628 #endif /* !__HAVE_FAST_SOFTINTS */ 629 630 /* 631 * If on the CPU and we have gotten this far, then we must yield. 632 */ 633 if (l->l_stat == LSONPROC && l != newl) { 634 KASSERT(lwp_locked(l, spc->spc_lwplock)); 635 KASSERT((l->l_flag & LW_IDLE) == 0); 636 l->l_stat = LSRUN; 637 lwp_setlock(l, spc->spc_mutex); 638 sched_enqueue(l); 639 sched_preempted(l); 640 641 /* 642 * Handle migration. Note that "migrating LWP" may 643 * be reset here, if interrupt/preemption happens 644 * early in idle LWP. 645 */ 646 if (l->l_target_cpu != NULL && (l->l_pflag & LP_BOUND) == 0) { 647 KASSERT((l->l_pflag & LP_INTR) == 0); 648 spc->spc_migrating = l; 649 } 650 } 651 652 /* Pick new LWP to run. */ 653 if (newl == NULL) { 654 newl = nextlwp(ci, spc); 655 } 656 657 /* Items that must be updated with the CPU locked. */ 658 if (!returning) { 659 /* Count time spent in current system call */ 660 SYSCALL_TIME_SLEEP(l); 661 662 updatertime(l, &bt); 663 664 /* Update the new LWP's start time. */ 665 newl->l_stime = bt; 666 667 /* 668 * ci_curlwp changes when a fast soft interrupt occurs. 669 * We use ci_onproc to keep track of which kernel or 670 * user thread is running 'underneath' the software 671 * interrupt. This is important for time accounting, 672 * itimers and forcing user threads to preempt (aston). 673 */ 674 ci->ci_onproc = newl; 675 } 676 677 /* 678 * Preemption related tasks. Must be done holding spc_mutex. Clear 679 * l_dopreempt without an atomic - it's only ever set non-zero by 680 * sched_resched_cpu() which also holds spc_mutex, and only ever 681 * cleared by the LWP itself (us) with atomics when not under lock. 682 */ 683 l->l_dopreempt = 0; 684 if (__predict_false(l->l_pfailaddr != 0)) { 685 LOCKSTAT_FLAG(lsflag); 686 LOCKSTAT_ENTER(lsflag); 687 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime); 688 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN, 689 1, l->l_pfailtime, l->l_pfailaddr); 690 LOCKSTAT_EXIT(lsflag); 691 l->l_pfailtime = 0; 692 l->l_pfaillock = 0; 693 l->l_pfailaddr = 0; 694 } 695 696 if (l != newl) { 697 struct lwp *prevlwp; 698 699 /* Release all locks, but leave the current LWP locked */ 700 if (l->l_mutex == spc->spc_mutex) { 701 /* 702 * Drop spc_lwplock, if the current LWP has been moved 703 * to the run queue (it is now locked by spc_mutex). 704 */ 705 mutex_spin_exit(spc->spc_lwplock); 706 } else { 707 /* 708 * Otherwise, drop the spc_mutex, we are done with the 709 * run queues. 710 */ 711 mutex_spin_exit(spc->spc_mutex); 712 } 713 714 /* We're down to only one lock, so do debug checks. */ 715 LOCKDEBUG_BARRIER(l->l_mutex, 1); 716 717 /* Count the context switch. */ 718 CPU_COUNT(CPU_COUNT_NSWTCH, 1); 719 l->l_ncsw++; 720 if ((l->l_pflag & LP_PREEMPTING) != 0) { 721 l->l_nivcsw++; 722 l->l_pflag &= ~LP_PREEMPTING; 723 } 724 725 /* 726 * Increase the count of spin-mutexes before the release 727 * of the last lock - we must remain at IPL_SCHED after 728 * releasing the lock. 729 */ 730 KASSERTMSG(ci->ci_mtx_count == -1, 731 "%s: cpu%u: ci_mtx_count (%d) != -1 " 732 "(block with spin-mutex held)", 733 __func__, cpu_index(ci), ci->ci_mtx_count); 734 oldspl = MUTEX_SPIN_OLDSPL(ci); 735 ci->ci_mtx_count = -2; 736 737 /* Update status for lwpctl, if present. */ 738 if (l->l_lwpctl != NULL) { 739 l->l_lwpctl->lc_curcpu = (l->l_stat == LSZOMB ? 740 LWPCTL_CPU_EXITED : LWPCTL_CPU_NONE); 741 } 742 743 /* 744 * If curlwp is a soft interrupt LWP, there's nobody on the 745 * other side to unlock - we're returning into an assembly 746 * trampoline. Unlock now. This is safe because this is a 747 * kernel LWP and is bound to current CPU: the worst anyone 748 * else will do to it, is to put it back onto this CPU's run 749 * queue (and the CPU is busy here right now!). 750 */ 751 if (returning) { 752 /* Keep IPL_SCHED after this; MD code will fix up. */ 753 l->l_pflag &= ~LP_RUNNING; 754 lwp_unlock(l); 755 } else { 756 /* A normal LWP: save old VM context. */ 757 pmap_deactivate(l); 758 } 759 760 /* 761 * If DTrace has set the active vtime enum to anything 762 * other than INACTIVE (0), then it should have set the 763 * function to call. 764 */ 765 if (__predict_false(dtrace_vtime_active)) { 766 (*dtrace_vtime_switch_func)(newl); 767 } 768 769 /* 770 * We must ensure not to come here from inside a read section. 771 */ 772 KASSERT(pserialize_not_in_read_section()); 773 774 /* Switch to the new LWP.. */ 775 #ifdef MULTIPROCESSOR 776 KASSERT(curlwp == ci->ci_curlwp); 777 #endif 778 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 779 prevlwp = cpu_switchto(l, newl, returning); 780 ci = curcpu(); 781 #ifdef MULTIPROCESSOR 782 KASSERT(curlwp == ci->ci_curlwp); 783 #endif 784 KASSERTMSG(l == curlwp, "l %p curlwp %p prevlwp %p", 785 l, curlwp, prevlwp); 786 KASSERT(prevlwp != NULL); 787 KASSERT(l->l_cpu == ci); 788 KASSERT(ci->ci_mtx_count == -2); 789 790 /* 791 * Immediately mark the previous LWP as no longer running 792 * and unlock (to keep lock wait times short as possible). 793 * We'll still be at IPL_SCHED afterwards. If a zombie, 794 * don't touch after clearing LP_RUNNING as it could be 795 * reaped by another CPU. Issue a memory barrier to ensure 796 * this. 797 */ 798 KASSERT((prevlwp->l_pflag & LP_RUNNING) != 0); 799 lock = prevlwp->l_mutex; 800 if (__predict_false(prevlwp->l_stat == LSZOMB)) { 801 membar_sync(); 802 } 803 prevlwp->l_pflag &= ~LP_RUNNING; 804 mutex_spin_exit(lock); 805 806 /* 807 * Switched away - we have new curlwp. 808 * Restore VM context and IPL. 809 */ 810 pmap_activate(l); 811 pcu_switchpoint(l); 812 813 /* Update status for lwpctl, if present. */ 814 if (l->l_lwpctl != NULL) { 815 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); 816 l->l_lwpctl->lc_pctr++; 817 } 818 819 /* 820 * Normalize the spin mutex count and restore the previous 821 * SPL. Note that, unless the caller disabled preemption, 822 * we can be preempted at any time after this splx(). 823 */ 824 KASSERT(l->l_cpu == ci); 825 KASSERT(ci->ci_mtx_count == -1); 826 ci->ci_mtx_count = 0; 827 splx(oldspl); 828 } else { 829 /* Nothing to do - just unlock and return. */ 830 mutex_spin_exit(spc->spc_mutex); 831 l->l_pflag &= ~LP_PREEMPTING; 832 lwp_unlock(l); 833 } 834 835 KASSERT(l == curlwp); 836 KASSERT(l->l_stat == LSONPROC); 837 838 SYSCALL_TIME_WAKEUP(l); 839 LOCKDEBUG_BARRIER(NULL, 1); 840 } 841 842 /* 843 * setrunnable: change LWP state to be runnable, placing it on the run queue. 844 * 845 * Call with the process and LWP locked. Will return with the LWP unlocked. 846 */ 847 void 848 setrunnable(struct lwp *l) 849 { 850 struct proc *p = l->l_proc; 851 struct cpu_info *ci; 852 kmutex_t *oldlock; 853 854 KASSERT((l->l_flag & LW_IDLE) == 0); 855 KASSERT((l->l_flag & LW_DBGSUSPEND) == 0); 856 KASSERT(mutex_owned(p->p_lock)); 857 KASSERT(lwp_locked(l, NULL)); 858 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex); 859 860 switch (l->l_stat) { 861 case LSSTOP: 862 /* 863 * If we're being traced (possibly because someone attached us 864 * while we were stopped), check for a signal from the debugger. 865 */ 866 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xsig != 0) 867 signotify(l); 868 p->p_nrlwps++; 869 break; 870 case LSSUSPENDED: 871 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 872 l->l_flag &= ~LW_WSUSPEND; 873 p->p_nrlwps++; 874 cv_broadcast(&p->p_lwpcv); 875 break; 876 case LSSLEEP: 877 KASSERT(l->l_wchan != NULL); 878 break; 879 case LSIDL: 880 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 881 break; 882 default: 883 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 884 } 885 886 /* 887 * If the LWP was sleeping, start it again. 888 */ 889 if (l->l_wchan != NULL) { 890 l->l_stat = LSSLEEP; 891 /* lwp_unsleep() will release the lock. */ 892 lwp_unsleep(l, true); 893 return; 894 } 895 896 /* 897 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 898 * about to call mi_switch(), in which case it will yield. 899 */ 900 if ((l->l_pflag & LP_RUNNING) != 0) { 901 l->l_stat = LSONPROC; 902 l->l_slptime = 0; 903 lwp_unlock(l); 904 return; 905 } 906 907 /* 908 * Look for a CPU to run. 909 * Set the LWP runnable. 910 */ 911 ci = sched_takecpu(l); 912 l->l_cpu = ci; 913 spc_lock(ci); 914 oldlock = lwp_setlock(l, l->l_cpu->ci_schedstate.spc_mutex); 915 sched_setrunnable(l); 916 l->l_stat = LSRUN; 917 l->l_slptime = 0; 918 sched_enqueue(l); 919 sched_resched_lwp(l, true); 920 /* SPC & LWP now unlocked. */ 921 mutex_spin_exit(oldlock); 922 } 923 924 /* 925 * suspendsched: 926 * 927 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 928 */ 929 void 930 suspendsched(void) 931 { 932 CPU_INFO_ITERATOR cii; 933 struct cpu_info *ci; 934 struct lwp *l; 935 struct proc *p; 936 937 /* 938 * We do this by process in order not to violate the locking rules. 939 */ 940 mutex_enter(proc_lock); 941 PROCLIST_FOREACH(p, &allproc) { 942 mutex_enter(p->p_lock); 943 if ((p->p_flag & PK_SYSTEM) != 0) { 944 mutex_exit(p->p_lock); 945 continue; 946 } 947 948 if (p->p_stat != SSTOP) { 949 if (p->p_stat != SZOMB && p->p_stat != SDEAD) { 950 p->p_pptr->p_nstopchild++; 951 p->p_waited = 0; 952 } 953 p->p_stat = SSTOP; 954 } 955 956 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 957 if (l == curlwp) 958 continue; 959 960 lwp_lock(l); 961 962 /* 963 * Set L_WREBOOT so that the LWP will suspend itself 964 * when it tries to return to user mode. We want to 965 * try and get to get as many LWPs as possible to 966 * the user / kernel boundary, so that they will 967 * release any locks that they hold. 968 */ 969 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 970 971 if (l->l_stat == LSSLEEP && 972 (l->l_flag & LW_SINTR) != 0) { 973 /* setrunnable() will release the lock. */ 974 setrunnable(l); 975 continue; 976 } 977 978 lwp_unlock(l); 979 } 980 981 mutex_exit(p->p_lock); 982 } 983 mutex_exit(proc_lock); 984 985 /* 986 * Kick all CPUs to make them preempt any LWPs running in user mode. 987 * They'll trap into the kernel and suspend themselves in userret(). 988 * 989 * Unusually, we don't hold any other scheduler object locked, which 990 * would keep preemption off for sched_resched_cpu(), so disable it 991 * explicitly. 992 */ 993 kpreempt_disable(); 994 for (CPU_INFO_FOREACH(cii, ci)) { 995 spc_lock(ci); 996 sched_resched_cpu(ci, PRI_KERNEL, true); 997 /* spc now unlocked */ 998 } 999 kpreempt_enable(); 1000 } 1001 1002 /* 1003 * sched_unsleep: 1004 * 1005 * The is called when the LWP has not been awoken normally but instead 1006 * interrupted: for example, if the sleep timed out. Because of this, 1007 * it's not a valid action for running or idle LWPs. 1008 */ 1009 static void 1010 sched_unsleep(struct lwp *l, bool cleanup) 1011 { 1012 1013 lwp_unlock(l); 1014 panic("sched_unsleep"); 1015 } 1016 1017 static void 1018 sched_changepri(struct lwp *l, pri_t pri) 1019 { 1020 struct schedstate_percpu *spc; 1021 struct cpu_info *ci; 1022 1023 KASSERT(lwp_locked(l, NULL)); 1024 1025 ci = l->l_cpu; 1026 spc = &ci->ci_schedstate; 1027 1028 if (l->l_stat == LSRUN) { 1029 KASSERT(lwp_locked(l, spc->spc_mutex)); 1030 sched_dequeue(l); 1031 l->l_priority = pri; 1032 sched_enqueue(l); 1033 sched_resched_lwp(l, false); 1034 } else if (l->l_stat == LSONPROC && l->l_class != SCHED_OTHER) { 1035 /* On priority drop, only evict realtime LWPs. */ 1036 KASSERT(lwp_locked(l, spc->spc_lwplock)); 1037 l->l_priority = pri; 1038 spc_lock(ci); 1039 sched_resched_cpu(ci, spc->spc_maxpriority, true); 1040 /* spc now unlocked */ 1041 } else { 1042 l->l_priority = pri; 1043 } 1044 } 1045 1046 static void 1047 sched_lendpri(struct lwp *l, pri_t pri) 1048 { 1049 struct schedstate_percpu *spc; 1050 struct cpu_info *ci; 1051 1052 KASSERT(lwp_locked(l, NULL)); 1053 1054 ci = l->l_cpu; 1055 spc = &ci->ci_schedstate; 1056 1057 if (l->l_stat == LSRUN) { 1058 KASSERT(lwp_locked(l, spc->spc_mutex)); 1059 sched_dequeue(l); 1060 l->l_inheritedprio = pri; 1061 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1062 sched_enqueue(l); 1063 sched_resched_lwp(l, false); 1064 } else if (l->l_stat == LSONPROC && l->l_class != SCHED_OTHER) { 1065 /* On priority drop, only evict realtime LWPs. */ 1066 KASSERT(lwp_locked(l, spc->spc_lwplock)); 1067 l->l_inheritedprio = pri; 1068 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1069 spc_lock(ci); 1070 sched_resched_cpu(ci, spc->spc_maxpriority, true); 1071 /* spc now unlocked */ 1072 } else { 1073 l->l_inheritedprio = pri; 1074 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1075 } 1076 } 1077 1078 struct lwp * 1079 syncobj_noowner(wchan_t wchan) 1080 { 1081 1082 return NULL; 1083 } 1084 1085 /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */ 1086 const fixpt_t ccpu = 0.95122942450071400909 * FSCALE; 1087 1088 /* 1089 * Constants for averages over 1, 5 and 15 minutes when sampling at 1090 * 5 second intervals. 1091 */ 1092 static const fixpt_t cexp[ ] = { 1093 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 1094 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 1095 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 1096 }; 1097 1098 /* 1099 * sched_pstats: 1100 * 1101 * => Update process statistics and check CPU resource allocation. 1102 * => Call scheduler-specific hook to eventually adjust LWP priorities. 1103 * => Compute load average of a quantity on 1, 5 and 15 minute intervals. 1104 */ 1105 void 1106 sched_pstats(void) 1107 { 1108 extern struct loadavg averunnable; 1109 struct loadavg *avg = &averunnable; 1110 const int clkhz = (stathz != 0 ? stathz : hz); 1111 static bool backwards = false; 1112 static u_int lavg_count = 0; 1113 struct proc *p; 1114 int nrun; 1115 1116 sched_pstats_ticks++; 1117 if (++lavg_count >= 5) { 1118 lavg_count = 0; 1119 nrun = 0; 1120 } 1121 mutex_enter(proc_lock); 1122 PROCLIST_FOREACH(p, &allproc) { 1123 struct lwp *l; 1124 struct rlimit *rlim; 1125 time_t runtm; 1126 int sig; 1127 1128 /* Increment sleep time (if sleeping), ignore overflow. */ 1129 mutex_enter(p->p_lock); 1130 runtm = p->p_rtime.sec; 1131 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1132 fixpt_t lpctcpu; 1133 u_int lcpticks; 1134 1135 if (__predict_false((l->l_flag & LW_IDLE) != 0)) 1136 continue; 1137 lwp_lock(l); 1138 runtm += l->l_rtime.sec; 1139 l->l_swtime++; 1140 sched_lwp_stats(l); 1141 1142 /* For load average calculation. */ 1143 if (__predict_false(lavg_count == 0) && 1144 (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) { 1145 switch (l->l_stat) { 1146 case LSSLEEP: 1147 if (l->l_slptime > 1) { 1148 break; 1149 } 1150 /* FALLTHROUGH */ 1151 case LSRUN: 1152 case LSONPROC: 1153 case LSIDL: 1154 nrun++; 1155 } 1156 } 1157 lwp_unlock(l); 1158 1159 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 1160 if (l->l_slptime != 0) 1161 continue; 1162 1163 lpctcpu = l->l_pctcpu; 1164 lcpticks = atomic_swap_uint(&l->l_cpticks, 0); 1165 lpctcpu += ((FSCALE - ccpu) * 1166 (lcpticks * FSCALE / clkhz)) >> FSHIFT; 1167 l->l_pctcpu = lpctcpu; 1168 } 1169 /* Calculating p_pctcpu only for ps(1) */ 1170 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 1171 1172 if (__predict_false(runtm < 0)) { 1173 if (!backwards) { 1174 backwards = true; 1175 printf("WARNING: negative runtime; " 1176 "monotonic clock has gone backwards\n"); 1177 } 1178 mutex_exit(p->p_lock); 1179 continue; 1180 } 1181 1182 /* 1183 * Check if the process exceeds its CPU resource allocation. 1184 * If over the hard limit, kill it with SIGKILL. 1185 * If over the soft limit, send SIGXCPU and raise 1186 * the soft limit a little. 1187 */ 1188 rlim = &p->p_rlimit[RLIMIT_CPU]; 1189 sig = 0; 1190 if (__predict_false(runtm >= rlim->rlim_cur)) { 1191 if (runtm >= rlim->rlim_max) { 1192 sig = SIGKILL; 1193 log(LOG_NOTICE, 1194 "pid %d, command %s, is killed: %s\n", 1195 p->p_pid, p->p_comm, "exceeded RLIMIT_CPU"); 1196 uprintf("pid %d, command %s, is killed: %s\n", 1197 p->p_pid, p->p_comm, "exceeded RLIMIT_CPU"); 1198 } else { 1199 sig = SIGXCPU; 1200 if (rlim->rlim_cur < rlim->rlim_max) 1201 rlim->rlim_cur += 5; 1202 } 1203 } 1204 mutex_exit(p->p_lock); 1205 if (__predict_false(sig)) { 1206 KASSERT((p->p_flag & PK_SYSTEM) == 0); 1207 psignal(p, sig); 1208 } 1209 } 1210 1211 /* Load average calculation. */ 1212 if (__predict_false(lavg_count == 0)) { 1213 int i; 1214 CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg)); 1215 for (i = 0; i < __arraycount(cexp); i++) { 1216 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 1217 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 1218 } 1219 } 1220 1221 /* Lightning bolt. */ 1222 cv_broadcast(&lbolt); 1223 1224 mutex_exit(proc_lock); 1225 } 1226