1 /* $NetBSD: kern_synch.c,v 1.352 2022/10/26 23:23:28 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 11 * Daniel Sieger. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1990, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.352 2022/10/26 23:23:28 riastradh Exp $"); 73 74 #include "opt_kstack.h" 75 #include "opt_dtrace.h" 76 77 #define __MUTEX_PRIVATE 78 79 #include <sys/param.h> 80 #include <sys/systm.h> 81 #include <sys/proc.h> 82 #include <sys/kernel.h> 83 #include <sys/cpu.h> 84 #include <sys/pserialize.h> 85 #include <sys/resource.h> 86 #include <sys/resourcevar.h> 87 #include <sys/rwlock.h> 88 #include <sys/sched.h> 89 #include <sys/syscall_stats.h> 90 #include <sys/sleepq.h> 91 #include <sys/lockdebug.h> 92 #include <sys/evcnt.h> 93 #include <sys/intr.h> 94 #include <sys/lwpctl.h> 95 #include <sys/atomic.h> 96 #include <sys/syslog.h> 97 98 #include <uvm/uvm_extern.h> 99 100 #include <dev/lockstat.h> 101 102 #include <sys/dtrace_bsd.h> 103 int dtrace_vtime_active=0; 104 dtrace_vtime_switch_func_t dtrace_vtime_switch_func; 105 106 static void sched_unsleep(struct lwp *, bool); 107 static void sched_changepri(struct lwp *, pri_t); 108 static void sched_lendpri(struct lwp *, pri_t); 109 110 syncobj_t sleep_syncobj = { 111 .sobj_flag = SOBJ_SLEEPQ_SORTED, 112 .sobj_unsleep = sleepq_unsleep, 113 .sobj_changepri = sleepq_changepri, 114 .sobj_lendpri = sleepq_lendpri, 115 .sobj_owner = syncobj_noowner, 116 }; 117 118 syncobj_t sched_syncobj = { 119 .sobj_flag = SOBJ_SLEEPQ_SORTED, 120 .sobj_unsleep = sched_unsleep, 121 .sobj_changepri = sched_changepri, 122 .sobj_lendpri = sched_lendpri, 123 .sobj_owner = syncobj_noowner, 124 }; 125 126 syncobj_t kpause_syncobj = { 127 .sobj_flag = SOBJ_SLEEPQ_NULL, 128 .sobj_unsleep = sleepq_unsleep, 129 .sobj_changepri = sleepq_changepri, 130 .sobj_lendpri = sleepq_lendpri, 131 .sobj_owner = syncobj_noowner, 132 }; 133 134 /* "Lightning bolt": once a second sleep address. */ 135 kcondvar_t lbolt __cacheline_aligned; 136 137 u_int sched_pstats_ticks __cacheline_aligned; 138 139 /* Preemption event counters. */ 140 static struct evcnt kpreempt_ev_crit __cacheline_aligned; 141 static struct evcnt kpreempt_ev_klock __cacheline_aligned; 142 static struct evcnt kpreempt_ev_immed __cacheline_aligned; 143 144 void 145 synch_init(void) 146 { 147 148 cv_init(&lbolt, "lbolt"); 149 150 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL, 151 "kpreempt", "defer: critical section"); 152 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL, 153 "kpreempt", "defer: kernel_lock"); 154 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL, 155 "kpreempt", "immediate"); 156 } 157 158 /* 159 * OBSOLETE INTERFACE 160 * 161 * General sleep call. Suspends the current LWP until a wakeup is 162 * performed on the specified identifier. The LWP will then be made 163 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 164 * means no timeout). If pri includes PCATCH flag, signals are checked 165 * before and after sleeping, else signals are not checked. Returns 0 if 166 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 167 * signal needs to be delivered, ERESTART is returned if the current system 168 * call should be restarted if possible, and EINTR is returned if the system 169 * call should be interrupted by the signal (return EINTR). 170 */ 171 int 172 tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo) 173 { 174 struct lwp *l = curlwp; 175 sleepq_t *sq; 176 kmutex_t *mp; 177 bool catch_p; 178 179 KASSERT((l->l_pflag & LP_INTR) == 0); 180 KASSERT(ident != &lbolt); 181 182 if (sleepq_dontsleep(l)) { 183 (void)sleepq_abort(NULL, 0); 184 return 0; 185 } 186 187 l->l_kpriority = true; 188 catch_p = priority & PCATCH; 189 sq = sleeptab_lookup(&sleeptab, ident, &mp); 190 sleepq_enter(sq, l, mp); 191 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj, catch_p); 192 return sleepq_block(timo, catch_p, &sleep_syncobj); 193 } 194 195 int 196 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 197 kmutex_t *mtx) 198 { 199 struct lwp *l = curlwp; 200 sleepq_t *sq; 201 kmutex_t *mp; 202 bool catch_p; 203 int error; 204 205 KASSERT((l->l_pflag & LP_INTR) == 0); 206 KASSERT(ident != &lbolt); 207 208 if (sleepq_dontsleep(l)) { 209 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 210 return 0; 211 } 212 213 l->l_kpriority = true; 214 catch_p = priority & PCATCH; 215 sq = sleeptab_lookup(&sleeptab, ident, &mp); 216 sleepq_enter(sq, l, mp); 217 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj, catch_p); 218 mutex_exit(mtx); 219 error = sleepq_block(timo, catch_p, &sleep_syncobj); 220 221 if ((priority & PNORELOCK) == 0) 222 mutex_enter(mtx); 223 224 return error; 225 } 226 227 /* 228 * General sleep call for situations where a wake-up is not expected. 229 */ 230 int 231 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 232 { 233 struct lwp *l = curlwp; 234 int error; 235 236 KASSERT(!(timo == 0 && intr == false)); 237 238 if (sleepq_dontsleep(l)) 239 return sleepq_abort(NULL, 0); 240 241 if (mtx != NULL) 242 mutex_exit(mtx); 243 l->l_kpriority = true; 244 lwp_lock(l); 245 KERNEL_UNLOCK_ALL(NULL, &l->l_biglocks); 246 sleepq_enqueue(NULL, l, wmesg, &kpause_syncobj, intr); 247 error = sleepq_block(timo, intr, &kpause_syncobj); 248 if (mtx != NULL) 249 mutex_enter(mtx); 250 251 return error; 252 } 253 254 /* 255 * OBSOLETE INTERFACE 256 * 257 * Make all LWPs sleeping on the specified identifier runnable. 258 */ 259 void 260 wakeup(wchan_t ident) 261 { 262 sleepq_t *sq; 263 kmutex_t *mp; 264 265 if (__predict_false(cold)) 266 return; 267 268 sq = sleeptab_lookup(&sleeptab, ident, &mp); 269 sleepq_wake(sq, ident, (u_int)-1, mp); 270 } 271 272 /* 273 * General yield call. Puts the current LWP back on its run queue and 274 * performs a context switch. 275 */ 276 void 277 yield(void) 278 { 279 struct lwp *l = curlwp; 280 281 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 282 lwp_lock(l); 283 284 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 285 KASSERT(l->l_stat == LSONPROC); 286 287 /* Voluntary - ditch kpriority boost. */ 288 l->l_kpriority = false; 289 spc_lock(l->l_cpu); 290 mi_switch(l); 291 KERNEL_LOCK(l->l_biglocks, l); 292 } 293 294 /* 295 * General preemption call. Puts the current LWP back on its run queue 296 * and performs an involuntary context switch. Different from yield() 297 * in that: 298 * 299 * - It's counted differently (involuntary vs. voluntary). 300 * - Realtime threads go to the head of their runqueue vs. tail for yield(). 301 * - Priority boost is retained unless LWP has exceeded timeslice. 302 */ 303 void 304 preempt(void) 305 { 306 struct lwp *l = curlwp; 307 308 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 309 lwp_lock(l); 310 311 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 312 KASSERT(l->l_stat == LSONPROC); 313 314 spc_lock(l->l_cpu); 315 /* Involuntary - keep kpriority boost unless a CPU hog. */ 316 if ((l->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) != 0) { 317 l->l_kpriority = false; 318 } 319 l->l_pflag |= LP_PREEMPTING; 320 mi_switch(l); 321 KERNEL_LOCK(l->l_biglocks, l); 322 } 323 324 /* 325 * Return true if the current LWP should yield the processor. Intended to 326 * be used by long-running code in kernel. 327 */ 328 inline bool 329 preempt_needed(void) 330 { 331 lwp_t *l = curlwp; 332 int needed; 333 334 KPREEMPT_DISABLE(l); 335 needed = l->l_cpu->ci_want_resched; 336 KPREEMPT_ENABLE(l); 337 338 return (needed != 0); 339 } 340 341 /* 342 * A breathing point for long running code in kernel. 343 */ 344 void 345 preempt_point(void) 346 { 347 348 if (__predict_false(preempt_needed())) { 349 preempt(); 350 } 351 } 352 353 /* 354 * Handle a request made by another agent to preempt the current LWP 355 * in-kernel. Usually called when l_dopreempt may be non-zero. 356 * 357 * Character addresses for lockstat only. 358 */ 359 static char kpreempt_is_disabled; 360 static char kernel_lock_held; 361 static char is_softint_lwp; 362 static char spl_is_raised; 363 364 bool 365 kpreempt(uintptr_t where) 366 { 367 uintptr_t failed; 368 lwp_t *l; 369 int s, dop, lsflag; 370 371 l = curlwp; 372 failed = 0; 373 while ((dop = l->l_dopreempt) != 0) { 374 if (l->l_stat != LSONPROC) { 375 /* 376 * About to block (or die), let it happen. 377 * Doesn't really count as "preemption has 378 * been blocked", since we're going to 379 * context switch. 380 */ 381 atomic_swap_uint(&l->l_dopreempt, 0); 382 return true; 383 } 384 KASSERT((l->l_flag & LW_IDLE) == 0); 385 if (__predict_false(l->l_nopreempt != 0)) { 386 /* LWP holds preemption disabled, explicitly. */ 387 if ((dop & DOPREEMPT_COUNTED) == 0) { 388 kpreempt_ev_crit.ev_count++; 389 } 390 failed = (uintptr_t)&kpreempt_is_disabled; 391 break; 392 } 393 if (__predict_false((l->l_pflag & LP_INTR) != 0)) { 394 /* Can't preempt soft interrupts yet. */ 395 atomic_swap_uint(&l->l_dopreempt, 0); 396 failed = (uintptr_t)&is_softint_lwp; 397 break; 398 } 399 s = splsched(); 400 if (__predict_false(l->l_blcnt != 0 || 401 curcpu()->ci_biglock_wanted != NULL)) { 402 /* Hold or want kernel_lock, code is not MT safe. */ 403 splx(s); 404 if ((dop & DOPREEMPT_COUNTED) == 0) { 405 kpreempt_ev_klock.ev_count++; 406 } 407 failed = (uintptr_t)&kernel_lock_held; 408 break; 409 } 410 if (__predict_false(!cpu_kpreempt_enter(where, s))) { 411 /* 412 * It may be that the IPL is too high. 413 * kpreempt_enter() can schedule an 414 * interrupt to retry later. 415 */ 416 splx(s); 417 failed = (uintptr_t)&spl_is_raised; 418 break; 419 } 420 /* Do it! */ 421 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) { 422 kpreempt_ev_immed.ev_count++; 423 } 424 lwp_lock(l); 425 /* Involuntary - keep kpriority boost. */ 426 l->l_pflag |= LP_PREEMPTING; 427 spc_lock(l->l_cpu); 428 mi_switch(l); 429 l->l_nopreempt++; 430 splx(s); 431 432 /* Take care of any MD cleanup. */ 433 cpu_kpreempt_exit(where); 434 l->l_nopreempt--; 435 } 436 437 if (__predict_true(!failed)) { 438 return false; 439 } 440 441 /* Record preemption failure for reporting via lockstat. */ 442 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED); 443 lsflag = 0; 444 LOCKSTAT_ENTER(lsflag); 445 if (__predict_false(lsflag)) { 446 if (where == 0) { 447 where = (uintptr_t)__builtin_return_address(0); 448 } 449 /* Preemption is on, might recurse, so make it atomic. */ 450 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL, 451 (void *)where) == NULL) { 452 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime); 453 l->l_pfaillock = failed; 454 } 455 } 456 LOCKSTAT_EXIT(lsflag); 457 return true; 458 } 459 460 /* 461 * Return true if preemption is explicitly disabled. 462 */ 463 bool 464 kpreempt_disabled(void) 465 { 466 const lwp_t *l = curlwp; 467 468 return l->l_nopreempt != 0 || l->l_stat == LSZOMB || 469 (l->l_flag & LW_IDLE) != 0 || (l->l_pflag & LP_INTR) != 0 || 470 cpu_kpreempt_disabled(); 471 } 472 473 /* 474 * Disable kernel preemption. 475 */ 476 void 477 kpreempt_disable(void) 478 { 479 480 KPREEMPT_DISABLE(curlwp); 481 } 482 483 /* 484 * Reenable kernel preemption. 485 */ 486 void 487 kpreempt_enable(void) 488 { 489 490 KPREEMPT_ENABLE(curlwp); 491 } 492 493 /* 494 * Compute the amount of time during which the current lwp was running. 495 * 496 * - update l_rtime unless it's an idle lwp. 497 */ 498 499 void 500 updatertime(lwp_t *l, const struct bintime *now) 501 { 502 503 if (__predict_false(l->l_flag & LW_IDLE)) 504 return; 505 506 /* rtime += now - stime */ 507 bintime_add(&l->l_rtime, now); 508 bintime_sub(&l->l_rtime, &l->l_stime); 509 } 510 511 /* 512 * Select next LWP from the current CPU to run.. 513 */ 514 static inline lwp_t * 515 nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc) 516 { 517 lwp_t *newl; 518 519 /* 520 * Let sched_nextlwp() select the LWP to run the CPU next. 521 * If no LWP is runnable, select the idle LWP. 522 * 523 * On arrival here LWPs on a run queue are locked by spc_mutex which 524 * is currently held. Idle LWPs are always locked by spc_lwplock, 525 * which may or may not be held here. On exit from this code block, 526 * in all cases newl is locked by spc_lwplock. 527 */ 528 newl = sched_nextlwp(); 529 if (newl != NULL) { 530 sched_dequeue(newl); 531 KASSERT(lwp_locked(newl, spc->spc_mutex)); 532 KASSERT(newl->l_cpu == ci); 533 newl->l_stat = LSONPROC; 534 newl->l_pflag |= LP_RUNNING; 535 spc->spc_curpriority = lwp_eprio(newl); 536 spc->spc_flags &= ~(SPCF_SWITCHCLEAR | SPCF_IDLE); 537 lwp_setlock(newl, spc->spc_lwplock); 538 } else { 539 /* 540 * The idle LWP does not get set to LSONPROC, because 541 * otherwise it screws up the output from top(1) etc. 542 */ 543 newl = ci->ci_data.cpu_idlelwp; 544 newl->l_pflag |= LP_RUNNING; 545 spc->spc_curpriority = PRI_IDLE; 546 spc->spc_flags = (spc->spc_flags & ~SPCF_SWITCHCLEAR) | 547 SPCF_IDLE; 548 } 549 550 /* 551 * Only clear want_resched if there are no pending (slow) software 552 * interrupts. We can do this without an atomic, because no new 553 * LWPs can appear in the queue due to our hold on spc_mutex, and 554 * the update to ci_want_resched will become globally visible before 555 * the release of spc_mutex becomes globally visible. 556 */ 557 ci->ci_want_resched = ci->ci_data.cpu_softints; 558 559 return newl; 560 } 561 562 /* 563 * The machine independent parts of context switch. 564 * 565 * NOTE: l->l_cpu is not changed in this routine, because an LWP never 566 * changes its own l_cpu (that would screw up curcpu on many ports and could 567 * cause all kinds of other evil stuff). l_cpu is always changed by some 568 * other actor, when it's known the LWP is not running (the LP_RUNNING flag 569 * is checked under lock). 570 */ 571 void 572 mi_switch(lwp_t *l) 573 { 574 struct cpu_info *ci; 575 struct schedstate_percpu *spc; 576 struct lwp *newl; 577 kmutex_t *lock; 578 int oldspl; 579 struct bintime bt; 580 bool returning; 581 582 KASSERT(lwp_locked(l, NULL)); 583 KASSERT(kpreempt_disabled()); 584 KASSERT(mutex_owned(curcpu()->ci_schedstate.spc_mutex)); 585 KASSERTMSG(l->l_blcnt == 0, "kernel_lock leaked"); 586 587 kstack_check_magic(l); 588 589 binuptime(&bt); 590 591 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 592 KASSERT((l->l_pflag & LP_RUNNING) != 0); 593 KASSERT(l->l_cpu == curcpu() || l->l_stat == LSRUN); 594 ci = curcpu(); 595 spc = &ci->ci_schedstate; 596 returning = false; 597 newl = NULL; 598 599 /* 600 * If we have been asked to switch to a specific LWP, then there 601 * is no need to inspect the run queues. If a soft interrupt is 602 * blocking, then return to the interrupted thread without adjusting 603 * VM context or its start time: neither have been changed in order 604 * to take the interrupt. 605 */ 606 if (l->l_switchto != NULL) { 607 if ((l->l_pflag & LP_INTR) != 0) { 608 returning = true; 609 softint_block(l); 610 if ((l->l_pflag & LP_TIMEINTR) != 0) 611 updatertime(l, &bt); 612 } 613 newl = l->l_switchto; 614 l->l_switchto = NULL; 615 } 616 #ifndef __HAVE_FAST_SOFTINTS 617 else if (ci->ci_data.cpu_softints != 0) { 618 /* There are pending soft interrupts, so pick one. */ 619 newl = softint_picklwp(); 620 newl->l_stat = LSONPROC; 621 newl->l_pflag |= LP_RUNNING; 622 } 623 #endif /* !__HAVE_FAST_SOFTINTS */ 624 625 /* 626 * If on the CPU and we have gotten this far, then we must yield. 627 */ 628 if (l->l_stat == LSONPROC && l != newl) { 629 KASSERT(lwp_locked(l, spc->spc_lwplock)); 630 KASSERT((l->l_flag & LW_IDLE) == 0); 631 l->l_stat = LSRUN; 632 lwp_setlock(l, spc->spc_mutex); 633 sched_enqueue(l); 634 sched_preempted(l); 635 636 /* 637 * Handle migration. Note that "migrating LWP" may 638 * be reset here, if interrupt/preemption happens 639 * early in idle LWP. 640 */ 641 if (l->l_target_cpu != NULL && (l->l_pflag & LP_BOUND) == 0) { 642 KASSERT((l->l_pflag & LP_INTR) == 0); 643 spc->spc_migrating = l; 644 } 645 } 646 647 /* Pick new LWP to run. */ 648 if (newl == NULL) { 649 newl = nextlwp(ci, spc); 650 } 651 652 /* Items that must be updated with the CPU locked. */ 653 if (!returning) { 654 /* Count time spent in current system call */ 655 SYSCALL_TIME_SLEEP(l); 656 657 updatertime(l, &bt); 658 659 /* Update the new LWP's start time. */ 660 newl->l_stime = bt; 661 662 /* 663 * ci_curlwp changes when a fast soft interrupt occurs. 664 * We use ci_onproc to keep track of which kernel or 665 * user thread is running 'underneath' the software 666 * interrupt. This is important for time accounting, 667 * itimers and forcing user threads to preempt (aston). 668 */ 669 ci->ci_onproc = newl; 670 } 671 672 /* 673 * Preemption related tasks. Must be done holding spc_mutex. Clear 674 * l_dopreempt without an atomic - it's only ever set non-zero by 675 * sched_resched_cpu() which also holds spc_mutex, and only ever 676 * cleared by the LWP itself (us) with atomics when not under lock. 677 */ 678 l->l_dopreempt = 0; 679 if (__predict_false(l->l_pfailaddr != 0)) { 680 LOCKSTAT_FLAG(lsflag); 681 LOCKSTAT_ENTER(lsflag); 682 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime); 683 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN, 684 1, l->l_pfailtime, l->l_pfailaddr); 685 LOCKSTAT_EXIT(lsflag); 686 l->l_pfailtime = 0; 687 l->l_pfaillock = 0; 688 l->l_pfailaddr = 0; 689 } 690 691 if (l != newl) { 692 struct lwp *prevlwp; 693 694 /* Release all locks, but leave the current LWP locked */ 695 if (l->l_mutex == spc->spc_mutex) { 696 /* 697 * Drop spc_lwplock, if the current LWP has been moved 698 * to the run queue (it is now locked by spc_mutex). 699 */ 700 mutex_spin_exit(spc->spc_lwplock); 701 } else { 702 /* 703 * Otherwise, drop the spc_mutex, we are done with the 704 * run queues. 705 */ 706 mutex_spin_exit(spc->spc_mutex); 707 } 708 709 /* We're down to only one lock, so do debug checks. */ 710 LOCKDEBUG_BARRIER(l->l_mutex, 1); 711 712 /* Count the context switch. */ 713 CPU_COUNT(CPU_COUNT_NSWTCH, 1); 714 l->l_ncsw++; 715 if ((l->l_pflag & LP_PREEMPTING) != 0) { 716 l->l_nivcsw++; 717 l->l_pflag &= ~LP_PREEMPTING; 718 } 719 720 /* 721 * Increase the count of spin-mutexes before the release 722 * of the last lock - we must remain at IPL_SCHED after 723 * releasing the lock. 724 */ 725 KASSERTMSG(ci->ci_mtx_count == -1, 726 "%s: cpu%u: ci_mtx_count (%d) != -1 " 727 "(block with spin-mutex held)", 728 __func__, cpu_index(ci), ci->ci_mtx_count); 729 oldspl = MUTEX_SPIN_OLDSPL(ci); 730 ci->ci_mtx_count = -2; 731 732 /* Update status for lwpctl, if present. */ 733 if (l->l_lwpctl != NULL) { 734 l->l_lwpctl->lc_curcpu = (l->l_stat == LSZOMB ? 735 LWPCTL_CPU_EXITED : LWPCTL_CPU_NONE); 736 } 737 738 /* 739 * If curlwp is a soft interrupt LWP, there's nobody on the 740 * other side to unlock - we're returning into an assembly 741 * trampoline. Unlock now. This is safe because this is a 742 * kernel LWP and is bound to current CPU: the worst anyone 743 * else will do to it, is to put it back onto this CPU's run 744 * queue (and the CPU is busy here right now!). 745 */ 746 if (returning) { 747 /* Keep IPL_SCHED after this; MD code will fix up. */ 748 l->l_pflag &= ~LP_RUNNING; 749 lwp_unlock(l); 750 } else { 751 /* A normal LWP: save old VM context. */ 752 pmap_deactivate(l); 753 } 754 755 /* 756 * If DTrace has set the active vtime enum to anything 757 * other than INACTIVE (0), then it should have set the 758 * function to call. 759 */ 760 if (__predict_false(dtrace_vtime_active)) { 761 (*dtrace_vtime_switch_func)(newl); 762 } 763 764 /* 765 * We must ensure not to come here from inside a read section. 766 */ 767 KASSERT(pserialize_not_in_read_section()); 768 769 /* Switch to the new LWP.. */ 770 #ifdef MULTIPROCESSOR 771 KASSERT(curlwp == ci->ci_curlwp); 772 #endif 773 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 774 prevlwp = cpu_switchto(l, newl, returning); 775 ci = curcpu(); 776 #ifdef MULTIPROCESSOR 777 KASSERT(curlwp == ci->ci_curlwp); 778 #endif 779 KASSERTMSG(l == curlwp, "l %p curlwp %p prevlwp %p", 780 l, curlwp, prevlwp); 781 KASSERT(prevlwp != NULL); 782 KASSERT(l->l_cpu == ci); 783 KASSERT(ci->ci_mtx_count == -2); 784 785 /* 786 * Immediately mark the previous LWP as no longer running 787 * and unlock (to keep lock wait times short as possible). 788 * We'll still be at IPL_SCHED afterwards. If a zombie, 789 * don't touch after clearing LP_RUNNING as it could be 790 * reaped by another CPU. Issue a memory barrier to ensure 791 * this. 792 * 793 * atomic_store_release matches atomic_load_acquire in 794 * lwp_free. 795 */ 796 KASSERT((prevlwp->l_pflag & LP_RUNNING) != 0); 797 lock = prevlwp->l_mutex; 798 if (__predict_false(prevlwp->l_stat == LSZOMB)) { 799 atomic_store_release(&prevlwp->l_pflag, 800 prevlwp->l_pflag & ~LP_RUNNING); 801 } else { 802 prevlwp->l_pflag &= ~LP_RUNNING; 803 } 804 mutex_spin_exit(lock); 805 806 /* 807 * Switched away - we have new curlwp. 808 * Restore VM context and IPL. 809 */ 810 pmap_activate(l); 811 pcu_switchpoint(l); 812 813 /* Update status for lwpctl, if present. */ 814 if (l->l_lwpctl != NULL) { 815 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); 816 l->l_lwpctl->lc_pctr++; 817 } 818 819 /* 820 * Normalize the spin mutex count and restore the previous 821 * SPL. Note that, unless the caller disabled preemption, 822 * we can be preempted at any time after this splx(). 823 */ 824 KASSERT(l->l_cpu == ci); 825 KASSERT(ci->ci_mtx_count == -1); 826 ci->ci_mtx_count = 0; 827 splx(oldspl); 828 } else { 829 /* Nothing to do - just unlock and return. */ 830 mutex_spin_exit(spc->spc_mutex); 831 l->l_pflag &= ~LP_PREEMPTING; 832 lwp_unlock(l); 833 } 834 835 KASSERT(l == curlwp); 836 KASSERT(l->l_stat == LSONPROC || (l->l_flag & LW_IDLE) != 0); 837 838 SYSCALL_TIME_WAKEUP(l); 839 LOCKDEBUG_BARRIER(NULL, 1); 840 } 841 842 /* 843 * setrunnable: change LWP state to be runnable, placing it on the run queue. 844 * 845 * Call with the process and LWP locked. Will return with the LWP unlocked. 846 */ 847 void 848 setrunnable(struct lwp *l) 849 { 850 struct proc *p = l->l_proc; 851 struct cpu_info *ci; 852 kmutex_t *oldlock; 853 854 KASSERT((l->l_flag & LW_IDLE) == 0); 855 KASSERT((l->l_flag & LW_DBGSUSPEND) == 0); 856 KASSERT(mutex_owned(p->p_lock)); 857 KASSERT(lwp_locked(l, NULL)); 858 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex); 859 860 switch (l->l_stat) { 861 case LSSTOP: 862 /* 863 * If we're being traced (possibly because someone attached us 864 * while we were stopped), check for a signal from the debugger. 865 */ 866 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xsig != 0) 867 signotify(l); 868 p->p_nrlwps++; 869 break; 870 case LSSUSPENDED: 871 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 872 l->l_flag &= ~LW_WSUSPEND; 873 p->p_nrlwps++; 874 cv_broadcast(&p->p_lwpcv); 875 break; 876 case LSSLEEP: 877 KASSERT(l->l_wchan != NULL); 878 break; 879 case LSIDL: 880 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 881 break; 882 default: 883 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 884 } 885 886 /* 887 * If the LWP was sleeping, start it again. 888 */ 889 if (l->l_wchan != NULL) { 890 l->l_stat = LSSLEEP; 891 /* lwp_unsleep() will release the lock. */ 892 lwp_unsleep(l, true); 893 return; 894 } 895 896 /* 897 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 898 * about to call mi_switch(), in which case it will yield. 899 */ 900 if ((l->l_pflag & LP_RUNNING) != 0) { 901 l->l_stat = LSONPROC; 902 l->l_slptime = 0; 903 lwp_unlock(l); 904 return; 905 } 906 907 /* 908 * Look for a CPU to run. 909 * Set the LWP runnable. 910 */ 911 ci = sched_takecpu(l); 912 l->l_cpu = ci; 913 spc_lock(ci); 914 oldlock = lwp_setlock(l, l->l_cpu->ci_schedstate.spc_mutex); 915 sched_setrunnable(l); 916 l->l_stat = LSRUN; 917 l->l_slptime = 0; 918 sched_enqueue(l); 919 sched_resched_lwp(l, true); 920 /* SPC & LWP now unlocked. */ 921 mutex_spin_exit(oldlock); 922 } 923 924 /* 925 * suspendsched: 926 * 927 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 928 */ 929 void 930 suspendsched(void) 931 { 932 CPU_INFO_ITERATOR cii; 933 struct cpu_info *ci; 934 struct lwp *l; 935 struct proc *p; 936 937 /* 938 * We do this by process in order not to violate the locking rules. 939 */ 940 mutex_enter(&proc_lock); 941 PROCLIST_FOREACH(p, &allproc) { 942 mutex_enter(p->p_lock); 943 if ((p->p_flag & PK_SYSTEM) != 0) { 944 mutex_exit(p->p_lock); 945 continue; 946 } 947 948 if (p->p_stat != SSTOP) { 949 if (p->p_stat != SZOMB && p->p_stat != SDEAD) { 950 p->p_pptr->p_nstopchild++; 951 p->p_waited = 0; 952 } 953 p->p_stat = SSTOP; 954 } 955 956 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 957 if (l == curlwp) 958 continue; 959 960 lwp_lock(l); 961 962 /* 963 * Set L_WREBOOT so that the LWP will suspend itself 964 * when it tries to return to user mode. We want to 965 * try and get to get as many LWPs as possible to 966 * the user / kernel boundary, so that they will 967 * release any locks that they hold. 968 */ 969 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 970 971 if (l->l_stat == LSSLEEP && 972 (l->l_flag & LW_SINTR) != 0) { 973 /* setrunnable() will release the lock. */ 974 setrunnable(l); 975 continue; 976 } 977 978 lwp_unlock(l); 979 } 980 981 mutex_exit(p->p_lock); 982 } 983 mutex_exit(&proc_lock); 984 985 /* 986 * Kick all CPUs to make them preempt any LWPs running in user mode. 987 * They'll trap into the kernel and suspend themselves in userret(). 988 * 989 * Unusually, we don't hold any other scheduler object locked, which 990 * would keep preemption off for sched_resched_cpu(), so disable it 991 * explicitly. 992 */ 993 kpreempt_disable(); 994 for (CPU_INFO_FOREACH(cii, ci)) { 995 spc_lock(ci); 996 sched_resched_cpu(ci, PRI_KERNEL, true); 997 /* spc now unlocked */ 998 } 999 kpreempt_enable(); 1000 } 1001 1002 /* 1003 * sched_unsleep: 1004 * 1005 * The is called when the LWP has not been awoken normally but instead 1006 * interrupted: for example, if the sleep timed out. Because of this, 1007 * it's not a valid action for running or idle LWPs. 1008 */ 1009 static void 1010 sched_unsleep(struct lwp *l, bool cleanup) 1011 { 1012 1013 lwp_unlock(l); 1014 panic("sched_unsleep"); 1015 } 1016 1017 static void 1018 sched_changepri(struct lwp *l, pri_t pri) 1019 { 1020 struct schedstate_percpu *spc; 1021 struct cpu_info *ci; 1022 1023 KASSERT(lwp_locked(l, NULL)); 1024 1025 ci = l->l_cpu; 1026 spc = &ci->ci_schedstate; 1027 1028 if (l->l_stat == LSRUN) { 1029 KASSERT(lwp_locked(l, spc->spc_mutex)); 1030 sched_dequeue(l); 1031 l->l_priority = pri; 1032 sched_enqueue(l); 1033 sched_resched_lwp(l, false); 1034 } else if (l->l_stat == LSONPROC && l->l_class != SCHED_OTHER) { 1035 /* On priority drop, only evict realtime LWPs. */ 1036 KASSERT(lwp_locked(l, spc->spc_lwplock)); 1037 l->l_priority = pri; 1038 spc_lock(ci); 1039 sched_resched_cpu(ci, spc->spc_maxpriority, true); 1040 /* spc now unlocked */ 1041 } else { 1042 l->l_priority = pri; 1043 } 1044 } 1045 1046 static void 1047 sched_lendpri(struct lwp *l, pri_t pri) 1048 { 1049 struct schedstate_percpu *spc; 1050 struct cpu_info *ci; 1051 1052 KASSERT(lwp_locked(l, NULL)); 1053 1054 ci = l->l_cpu; 1055 spc = &ci->ci_schedstate; 1056 1057 if (l->l_stat == LSRUN) { 1058 KASSERT(lwp_locked(l, spc->spc_mutex)); 1059 sched_dequeue(l); 1060 l->l_inheritedprio = pri; 1061 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1062 sched_enqueue(l); 1063 sched_resched_lwp(l, false); 1064 } else if (l->l_stat == LSONPROC && l->l_class != SCHED_OTHER) { 1065 /* On priority drop, only evict realtime LWPs. */ 1066 KASSERT(lwp_locked(l, spc->spc_lwplock)); 1067 l->l_inheritedprio = pri; 1068 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1069 spc_lock(ci); 1070 sched_resched_cpu(ci, spc->spc_maxpriority, true); 1071 /* spc now unlocked */ 1072 } else { 1073 l->l_inheritedprio = pri; 1074 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1075 } 1076 } 1077 1078 struct lwp * 1079 syncobj_noowner(wchan_t wchan) 1080 { 1081 1082 return NULL; 1083 } 1084 1085 /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */ 1086 const fixpt_t ccpu = 0.95122942450071400909 * FSCALE; 1087 1088 /* 1089 * Constants for averages over 1, 5 and 15 minutes when sampling at 1090 * 5 second intervals. 1091 */ 1092 static const fixpt_t cexp[ ] = { 1093 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 1094 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 1095 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 1096 }; 1097 1098 /* 1099 * sched_pstats: 1100 * 1101 * => Update process statistics and check CPU resource allocation. 1102 * => Call scheduler-specific hook to eventually adjust LWP priorities. 1103 * => Compute load average of a quantity on 1, 5 and 15 minute intervals. 1104 */ 1105 void 1106 sched_pstats(void) 1107 { 1108 struct loadavg *avg = &averunnable; 1109 const int clkhz = (stathz != 0 ? stathz : hz); 1110 static bool backwards = false; 1111 static u_int lavg_count = 0; 1112 struct proc *p; 1113 int nrun; 1114 1115 sched_pstats_ticks++; 1116 if (++lavg_count >= 5) { 1117 lavg_count = 0; 1118 nrun = 0; 1119 } 1120 mutex_enter(&proc_lock); 1121 PROCLIST_FOREACH(p, &allproc) { 1122 struct lwp *l; 1123 struct rlimit *rlim; 1124 time_t runtm; 1125 int sig; 1126 1127 /* Increment sleep time (if sleeping), ignore overflow. */ 1128 mutex_enter(p->p_lock); 1129 runtm = p->p_rtime.sec; 1130 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1131 fixpt_t lpctcpu; 1132 u_int lcpticks; 1133 1134 if (__predict_false((l->l_flag & LW_IDLE) != 0)) 1135 continue; 1136 lwp_lock(l); 1137 runtm += l->l_rtime.sec; 1138 l->l_swtime++; 1139 sched_lwp_stats(l); 1140 1141 /* For load average calculation. */ 1142 if (__predict_false(lavg_count == 0) && 1143 (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) { 1144 switch (l->l_stat) { 1145 case LSSLEEP: 1146 if (l->l_slptime > 1) { 1147 break; 1148 } 1149 /* FALLTHROUGH */ 1150 case LSRUN: 1151 case LSONPROC: 1152 case LSIDL: 1153 nrun++; 1154 } 1155 } 1156 lwp_unlock(l); 1157 1158 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 1159 if (l->l_slptime != 0) 1160 continue; 1161 1162 lpctcpu = l->l_pctcpu; 1163 lcpticks = atomic_swap_uint(&l->l_cpticks, 0); 1164 lpctcpu += ((FSCALE - ccpu) * 1165 (lcpticks * FSCALE / clkhz)) >> FSHIFT; 1166 l->l_pctcpu = lpctcpu; 1167 } 1168 /* Calculating p_pctcpu only for ps(1) */ 1169 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 1170 1171 if (__predict_false(runtm < 0)) { 1172 if (!backwards) { 1173 backwards = true; 1174 printf("WARNING: negative runtime; " 1175 "monotonic clock has gone backwards\n"); 1176 } 1177 mutex_exit(p->p_lock); 1178 continue; 1179 } 1180 1181 /* 1182 * Check if the process exceeds its CPU resource allocation. 1183 * If over the hard limit, kill it with SIGKILL. 1184 * If over the soft limit, send SIGXCPU and raise 1185 * the soft limit a little. 1186 */ 1187 rlim = &p->p_rlimit[RLIMIT_CPU]; 1188 sig = 0; 1189 if (__predict_false(runtm >= rlim->rlim_cur)) { 1190 if (runtm >= rlim->rlim_max) { 1191 sig = SIGKILL; 1192 log(LOG_NOTICE, 1193 "pid %d, command %s, is killed: %s\n", 1194 p->p_pid, p->p_comm, "exceeded RLIMIT_CPU"); 1195 uprintf("pid %d, command %s, is killed: %s\n", 1196 p->p_pid, p->p_comm, "exceeded RLIMIT_CPU"); 1197 } else { 1198 sig = SIGXCPU; 1199 if (rlim->rlim_cur < rlim->rlim_max) 1200 rlim->rlim_cur += 5; 1201 } 1202 } 1203 mutex_exit(p->p_lock); 1204 if (__predict_false(sig)) { 1205 KASSERT((p->p_flag & PK_SYSTEM) == 0); 1206 psignal(p, sig); 1207 } 1208 } 1209 1210 /* Load average calculation. */ 1211 if (__predict_false(lavg_count == 0)) { 1212 int i; 1213 CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg)); 1214 for (i = 0; i < __arraycount(cexp); i++) { 1215 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 1216 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 1217 } 1218 } 1219 1220 /* Lightning bolt. */ 1221 cv_broadcast(&lbolt); 1222 1223 mutex_exit(&proc_lock); 1224 } 1225