1 /* $NetBSD: kern_synch.c,v 1.316 2018/07/12 10:46:48 maxv Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 11 * Daniel Sieger. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1990, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.316 2018/07/12 10:46:48 maxv Exp $"); 73 74 #include "opt_kstack.h" 75 #include "opt_dtrace.h" 76 77 #define __MUTEX_PRIVATE 78 79 #include <sys/param.h> 80 #include <sys/systm.h> 81 #include <sys/proc.h> 82 #include <sys/kernel.h> 83 #include <sys/cpu.h> 84 #include <sys/pserialize.h> 85 #include <sys/resourcevar.h> 86 #include <sys/sched.h> 87 #include <sys/syscall_stats.h> 88 #include <sys/sleepq.h> 89 #include <sys/lockdebug.h> 90 #include <sys/evcnt.h> 91 #include <sys/intr.h> 92 #include <sys/lwpctl.h> 93 #include <sys/atomic.h> 94 #include <sys/syslog.h> 95 96 #include <uvm/uvm_extern.h> 97 98 #include <dev/lockstat.h> 99 100 #include <sys/dtrace_bsd.h> 101 int dtrace_vtime_active=0; 102 dtrace_vtime_switch_func_t dtrace_vtime_switch_func; 103 104 static void sched_unsleep(struct lwp *, bool); 105 static void sched_changepri(struct lwp *, pri_t); 106 static void sched_lendpri(struct lwp *, pri_t); 107 static void resched_cpu(struct lwp *); 108 109 syncobj_t sleep_syncobj = { 110 .sobj_flag = SOBJ_SLEEPQ_SORTED, 111 .sobj_unsleep = sleepq_unsleep, 112 .sobj_changepri = sleepq_changepri, 113 .sobj_lendpri = sleepq_lendpri, 114 .sobj_owner = syncobj_noowner, 115 }; 116 117 syncobj_t sched_syncobj = { 118 .sobj_flag = SOBJ_SLEEPQ_SORTED, 119 .sobj_unsleep = sched_unsleep, 120 .sobj_changepri = sched_changepri, 121 .sobj_lendpri = sched_lendpri, 122 .sobj_owner = syncobj_noowner, 123 }; 124 125 /* "Lightning bolt": once a second sleep address. */ 126 kcondvar_t lbolt __cacheline_aligned; 127 128 u_int sched_pstats_ticks __cacheline_aligned; 129 130 /* Preemption event counters. */ 131 static struct evcnt kpreempt_ev_crit __cacheline_aligned; 132 static struct evcnt kpreempt_ev_klock __cacheline_aligned; 133 static struct evcnt kpreempt_ev_immed __cacheline_aligned; 134 135 void 136 synch_init(void) 137 { 138 139 cv_init(&lbolt, "lbolt"); 140 141 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL, 142 "kpreempt", "defer: critical section"); 143 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL, 144 "kpreempt", "defer: kernel_lock"); 145 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL, 146 "kpreempt", "immediate"); 147 } 148 149 /* 150 * OBSOLETE INTERFACE 151 * 152 * General sleep call. Suspends the current LWP until a wakeup is 153 * performed on the specified identifier. The LWP will then be made 154 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 155 * means no timeout). If pri includes PCATCH flag, signals are checked 156 * before and after sleeping, else signals are not checked. Returns 0 if 157 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 158 * signal needs to be delivered, ERESTART is returned if the current system 159 * call should be restarted if possible, and EINTR is returned if the system 160 * call should be interrupted by the signal (return EINTR). 161 */ 162 int 163 tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo) 164 { 165 struct lwp *l = curlwp; 166 sleepq_t *sq; 167 kmutex_t *mp; 168 169 KASSERT((l->l_pflag & LP_INTR) == 0); 170 KASSERT(ident != &lbolt); 171 172 if (sleepq_dontsleep(l)) { 173 (void)sleepq_abort(NULL, 0); 174 return 0; 175 } 176 177 l->l_kpriority = true; 178 sq = sleeptab_lookup(&sleeptab, ident, &mp); 179 sleepq_enter(sq, l, mp); 180 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 181 return sleepq_block(timo, priority & PCATCH); 182 } 183 184 int 185 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 186 kmutex_t *mtx) 187 { 188 struct lwp *l = curlwp; 189 sleepq_t *sq; 190 kmutex_t *mp; 191 int error; 192 193 KASSERT((l->l_pflag & LP_INTR) == 0); 194 KASSERT(ident != &lbolt); 195 196 if (sleepq_dontsleep(l)) { 197 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 198 return 0; 199 } 200 201 l->l_kpriority = true; 202 sq = sleeptab_lookup(&sleeptab, ident, &mp); 203 sleepq_enter(sq, l, mp); 204 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 205 mutex_exit(mtx); 206 error = sleepq_block(timo, priority & PCATCH); 207 208 if ((priority & PNORELOCK) == 0) 209 mutex_enter(mtx); 210 211 return error; 212 } 213 214 /* 215 * General sleep call for situations where a wake-up is not expected. 216 */ 217 int 218 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 219 { 220 struct lwp *l = curlwp; 221 kmutex_t *mp; 222 sleepq_t *sq; 223 int error; 224 225 KASSERT(!(timo == 0 && intr == false)); 226 227 if (sleepq_dontsleep(l)) 228 return sleepq_abort(NULL, 0); 229 230 if (mtx != NULL) 231 mutex_exit(mtx); 232 l->l_kpriority = true; 233 sq = sleeptab_lookup(&sleeptab, l, &mp); 234 sleepq_enter(sq, l, mp); 235 sleepq_enqueue(sq, l, wmesg, &sleep_syncobj); 236 error = sleepq_block(timo, intr); 237 if (mtx != NULL) 238 mutex_enter(mtx); 239 240 return error; 241 } 242 243 /* 244 * OBSOLETE INTERFACE 245 * 246 * Make all LWPs sleeping on the specified identifier runnable. 247 */ 248 void 249 wakeup(wchan_t ident) 250 { 251 sleepq_t *sq; 252 kmutex_t *mp; 253 254 if (__predict_false(cold)) 255 return; 256 257 sq = sleeptab_lookup(&sleeptab, ident, &mp); 258 sleepq_wake(sq, ident, (u_int)-1, mp); 259 } 260 261 /* 262 * General yield call. Puts the current LWP back on its run queue and 263 * performs a voluntary context switch. Should only be called when the 264 * current LWP explicitly requests it (eg sched_yield(2)). 265 */ 266 void 267 yield(void) 268 { 269 struct lwp *l = curlwp; 270 271 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 272 lwp_lock(l); 273 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 274 KASSERT(l->l_stat == LSONPROC); 275 l->l_kpriority = false; 276 (void)mi_switch(l); 277 KERNEL_LOCK(l->l_biglocks, l); 278 } 279 280 /* 281 * General preemption call. Puts the current LWP back on its run queue 282 * and performs an involuntary context switch. 283 */ 284 void 285 preempt(void) 286 { 287 struct lwp *l = curlwp; 288 289 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 290 lwp_lock(l); 291 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 292 KASSERT(l->l_stat == LSONPROC); 293 l->l_kpriority = false; 294 l->l_nivcsw++; 295 (void)mi_switch(l); 296 KERNEL_LOCK(l->l_biglocks, l); 297 } 298 299 /* 300 * Handle a request made by another agent to preempt the current LWP 301 * in-kernel. Usually called when l_dopreempt may be non-zero. 302 * 303 * Character addresses for lockstat only. 304 */ 305 static char in_critical_section; 306 static char kernel_lock_held; 307 static char is_softint; 308 static char cpu_kpreempt_enter_fail; 309 310 bool 311 kpreempt(uintptr_t where) 312 { 313 uintptr_t failed; 314 lwp_t *l; 315 int s, dop, lsflag; 316 317 l = curlwp; 318 failed = 0; 319 while ((dop = l->l_dopreempt) != 0) { 320 if (l->l_stat != LSONPROC) { 321 /* 322 * About to block (or die), let it happen. 323 * Doesn't really count as "preemption has 324 * been blocked", since we're going to 325 * context switch. 326 */ 327 l->l_dopreempt = 0; 328 return true; 329 } 330 if (__predict_false((l->l_flag & LW_IDLE) != 0)) { 331 /* Can't preempt idle loop, don't count as failure. */ 332 l->l_dopreempt = 0; 333 return true; 334 } 335 if (__predict_false(l->l_nopreempt != 0)) { 336 /* LWP holds preemption disabled, explicitly. */ 337 if ((dop & DOPREEMPT_COUNTED) == 0) { 338 kpreempt_ev_crit.ev_count++; 339 } 340 failed = (uintptr_t)&in_critical_section; 341 break; 342 } 343 if (__predict_false((l->l_pflag & LP_INTR) != 0)) { 344 /* Can't preempt soft interrupts yet. */ 345 l->l_dopreempt = 0; 346 failed = (uintptr_t)&is_softint; 347 break; 348 } 349 s = splsched(); 350 if (__predict_false(l->l_blcnt != 0 || 351 curcpu()->ci_biglock_wanted != NULL)) { 352 /* Hold or want kernel_lock, code is not MT safe. */ 353 splx(s); 354 if ((dop & DOPREEMPT_COUNTED) == 0) { 355 kpreempt_ev_klock.ev_count++; 356 } 357 failed = (uintptr_t)&kernel_lock_held; 358 break; 359 } 360 if (__predict_false(!cpu_kpreempt_enter(where, s))) { 361 /* 362 * It may be that the IPL is too high. 363 * kpreempt_enter() can schedule an 364 * interrupt to retry later. 365 */ 366 splx(s); 367 failed = (uintptr_t)&cpu_kpreempt_enter_fail; 368 break; 369 } 370 /* Do it! */ 371 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) { 372 kpreempt_ev_immed.ev_count++; 373 } 374 lwp_lock(l); 375 mi_switch(l); 376 l->l_nopreempt++; 377 splx(s); 378 379 /* Take care of any MD cleanup. */ 380 cpu_kpreempt_exit(where); 381 l->l_nopreempt--; 382 } 383 384 if (__predict_true(!failed)) { 385 return false; 386 } 387 388 /* Record preemption failure for reporting via lockstat. */ 389 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED); 390 lsflag = 0; 391 LOCKSTAT_ENTER(lsflag); 392 if (__predict_false(lsflag)) { 393 if (where == 0) { 394 where = (uintptr_t)__builtin_return_address(0); 395 } 396 /* Preemption is on, might recurse, so make it atomic. */ 397 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL, 398 (void *)where) == NULL) { 399 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime); 400 l->l_pfaillock = failed; 401 } 402 } 403 LOCKSTAT_EXIT(lsflag); 404 return true; 405 } 406 407 /* 408 * Return true if preemption is explicitly disabled. 409 */ 410 bool 411 kpreempt_disabled(void) 412 { 413 const lwp_t *l = curlwp; 414 415 return l->l_nopreempt != 0 || l->l_stat == LSZOMB || 416 (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled(); 417 } 418 419 /* 420 * Disable kernel preemption. 421 */ 422 void 423 kpreempt_disable(void) 424 { 425 426 KPREEMPT_DISABLE(curlwp); 427 } 428 429 /* 430 * Reenable kernel preemption. 431 */ 432 void 433 kpreempt_enable(void) 434 { 435 436 KPREEMPT_ENABLE(curlwp); 437 } 438 439 /* 440 * Compute the amount of time during which the current lwp was running. 441 * 442 * - update l_rtime unless it's an idle lwp. 443 */ 444 445 void 446 updatertime(lwp_t *l, const struct bintime *now) 447 { 448 449 if (__predict_false(l->l_flag & LW_IDLE)) 450 return; 451 452 /* rtime += now - stime */ 453 bintime_add(&l->l_rtime, now); 454 bintime_sub(&l->l_rtime, &l->l_stime); 455 } 456 457 /* 458 * Select next LWP from the current CPU to run.. 459 */ 460 static inline lwp_t * 461 nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc) 462 { 463 lwp_t *newl; 464 465 /* 466 * Let sched_nextlwp() select the LWP to run the CPU next. 467 * If no LWP is runnable, select the idle LWP. 468 * 469 * Note that spc_lwplock might not necessary be held, and 470 * new thread would be unlocked after setting the LWP-lock. 471 */ 472 newl = sched_nextlwp(); 473 if (newl != NULL) { 474 sched_dequeue(newl); 475 KASSERT(lwp_locked(newl, spc->spc_mutex)); 476 KASSERT(newl->l_cpu == ci); 477 newl->l_stat = LSONPROC; 478 newl->l_pflag |= LP_RUNNING; 479 lwp_setlock(newl, spc->spc_lwplock); 480 } else { 481 newl = ci->ci_data.cpu_idlelwp; 482 newl->l_stat = LSONPROC; 483 newl->l_pflag |= LP_RUNNING; 484 } 485 486 /* 487 * Only clear want_resched if there are no pending (slow) 488 * software interrupts. 489 */ 490 ci->ci_want_resched = ci->ci_data.cpu_softints; 491 spc->spc_flags &= ~SPCF_SWITCHCLEAR; 492 spc->spc_curpriority = lwp_eprio(newl); 493 494 return newl; 495 } 496 497 /* 498 * The machine independent parts of context switch. 499 * 500 * Returns 1 if another LWP was actually run. 501 */ 502 int 503 mi_switch(lwp_t *l) 504 { 505 struct cpu_info *ci; 506 struct schedstate_percpu *spc; 507 struct lwp *newl; 508 int retval, oldspl; 509 struct bintime bt; 510 bool returning; 511 512 KASSERT(lwp_locked(l, NULL)); 513 KASSERT(kpreempt_disabled()); 514 LOCKDEBUG_BARRIER(l->l_mutex, 1); 515 516 kstack_check_magic(l); 517 518 binuptime(&bt); 519 520 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 521 KASSERT((l->l_pflag & LP_RUNNING) != 0); 522 KASSERT(l->l_cpu == curcpu()); 523 ci = l->l_cpu; 524 spc = &ci->ci_schedstate; 525 returning = false; 526 newl = NULL; 527 528 /* 529 * If we have been asked to switch to a specific LWP, then there 530 * is no need to inspect the run queues. If a soft interrupt is 531 * blocking, then return to the interrupted thread without adjusting 532 * VM context or its start time: neither have been changed in order 533 * to take the interrupt. 534 */ 535 if (l->l_switchto != NULL) { 536 if ((l->l_pflag & LP_INTR) != 0) { 537 returning = true; 538 softint_block(l); 539 if ((l->l_pflag & LP_TIMEINTR) != 0) 540 updatertime(l, &bt); 541 } 542 newl = l->l_switchto; 543 l->l_switchto = NULL; 544 } 545 #ifndef __HAVE_FAST_SOFTINTS 546 else if (ci->ci_data.cpu_softints != 0) { 547 /* There are pending soft interrupts, so pick one. */ 548 newl = softint_picklwp(); 549 newl->l_stat = LSONPROC; 550 newl->l_pflag |= LP_RUNNING; 551 } 552 #endif /* !__HAVE_FAST_SOFTINTS */ 553 554 /* Count time spent in current system call */ 555 if (!returning) { 556 SYSCALL_TIME_SLEEP(l); 557 558 updatertime(l, &bt); 559 } 560 561 /* Lock the runqueue */ 562 KASSERT(l->l_stat != LSRUN); 563 mutex_spin_enter(spc->spc_mutex); 564 565 /* 566 * If on the CPU and we have gotten this far, then we must yield. 567 */ 568 if (l->l_stat == LSONPROC && l != newl) { 569 KASSERT(lwp_locked(l, spc->spc_lwplock)); 570 if ((l->l_flag & LW_IDLE) == 0) { 571 l->l_stat = LSRUN; 572 lwp_setlock(l, spc->spc_mutex); 573 sched_enqueue(l, true); 574 /* 575 * Handle migration. Note that "migrating LWP" may 576 * be reset here, if interrupt/preemption happens 577 * early in idle LWP. 578 */ 579 if (l->l_target_cpu != NULL && 580 (l->l_pflag & LP_BOUND) == 0) { 581 KASSERT((l->l_pflag & LP_INTR) == 0); 582 spc->spc_migrating = l; 583 } 584 } else 585 l->l_stat = LSIDL; 586 } 587 588 /* Pick new LWP to run. */ 589 if (newl == NULL) { 590 newl = nextlwp(ci, spc); 591 } 592 593 /* Items that must be updated with the CPU locked. */ 594 if (!returning) { 595 /* Update the new LWP's start time. */ 596 newl->l_stime = bt; 597 598 /* 599 * ci_curlwp changes when a fast soft interrupt occurs. 600 * We use cpu_onproc to keep track of which kernel or 601 * user thread is running 'underneath' the software 602 * interrupt. This is important for time accounting, 603 * itimers and forcing user threads to preempt (aston). 604 */ 605 ci->ci_data.cpu_onproc = newl; 606 } 607 608 /* 609 * Preemption related tasks. Must be done with the current 610 * CPU locked. 611 */ 612 cpu_did_resched(l); 613 l->l_dopreempt = 0; 614 if (__predict_false(l->l_pfailaddr != 0)) { 615 LOCKSTAT_FLAG(lsflag); 616 LOCKSTAT_ENTER(lsflag); 617 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime); 618 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN, 619 1, l->l_pfailtime, l->l_pfailaddr); 620 LOCKSTAT_EXIT(lsflag); 621 l->l_pfailtime = 0; 622 l->l_pfaillock = 0; 623 l->l_pfailaddr = 0; 624 } 625 626 if (l != newl) { 627 struct lwp *prevlwp; 628 629 /* Release all locks, but leave the current LWP locked */ 630 if (l->l_mutex == spc->spc_mutex) { 631 /* 632 * Drop spc_lwplock, if the current LWP has been moved 633 * to the run queue (it is now locked by spc_mutex). 634 */ 635 mutex_spin_exit(spc->spc_lwplock); 636 } else { 637 /* 638 * Otherwise, drop the spc_mutex, we are done with the 639 * run queues. 640 */ 641 mutex_spin_exit(spc->spc_mutex); 642 } 643 644 /* 645 * Mark that context switch is going to be performed 646 * for this LWP, to protect it from being switched 647 * to on another CPU. 648 */ 649 KASSERT(l->l_ctxswtch == 0); 650 l->l_ctxswtch = 1; 651 l->l_ncsw++; 652 KASSERT((l->l_pflag & LP_RUNNING) != 0); 653 l->l_pflag &= ~LP_RUNNING; 654 655 /* 656 * Increase the count of spin-mutexes before the release 657 * of the last lock - we must remain at IPL_SCHED during 658 * the context switch. 659 */ 660 KASSERTMSG(ci->ci_mtx_count == -1, 661 "%s: cpu%u: ci_mtx_count (%d) != -1 " 662 "(block with spin-mutex held)", 663 __func__, cpu_index(ci), ci->ci_mtx_count); 664 oldspl = MUTEX_SPIN_OLDSPL(ci); 665 ci->ci_mtx_count--; 666 lwp_unlock(l); 667 668 /* Count the context switch on this CPU. */ 669 ci->ci_data.cpu_nswtch++; 670 671 /* Update status for lwpctl, if present. */ 672 if (l->l_lwpctl != NULL) 673 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE; 674 675 /* 676 * Save old VM context, unless a soft interrupt 677 * handler is blocking. 678 */ 679 if (!returning) 680 pmap_deactivate(l); 681 682 /* 683 * We may need to spin-wait if 'newl' is still 684 * context switching on another CPU. 685 */ 686 if (__predict_false(newl->l_ctxswtch != 0)) { 687 u_int count; 688 count = SPINLOCK_BACKOFF_MIN; 689 while (newl->l_ctxswtch) 690 SPINLOCK_BACKOFF(count); 691 } 692 693 /* 694 * If DTrace has set the active vtime enum to anything 695 * other than INACTIVE (0), then it should have set the 696 * function to call. 697 */ 698 if (__predict_false(dtrace_vtime_active)) { 699 (*dtrace_vtime_switch_func)(newl); 700 } 701 702 /* Switch to the new LWP.. */ 703 #ifdef MULTIPROCESSOR 704 KASSERT(curlwp == ci->ci_curlwp); 705 #endif 706 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 707 prevlwp = cpu_switchto(l, newl, returning); 708 ci = curcpu(); 709 #ifdef MULTIPROCESSOR 710 KASSERT(curlwp == ci->ci_curlwp); 711 #endif 712 KASSERTMSG(l == curlwp, "l %p curlwp %p prevlwp %p", 713 l, curlwp, prevlwp); 714 715 /* 716 * Switched away - we have new curlwp. 717 * Restore VM context and IPL. 718 */ 719 pmap_activate(l); 720 pcu_switchpoint(l); 721 722 if (prevlwp != NULL) { 723 /* Normalize the count of the spin-mutexes */ 724 ci->ci_mtx_count++; 725 /* Unmark the state of context switch */ 726 membar_exit(); 727 prevlwp->l_ctxswtch = 0; 728 } 729 730 /* Update status for lwpctl, if present. */ 731 if (l->l_lwpctl != NULL) { 732 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); 733 l->l_lwpctl->lc_pctr++; 734 } 735 736 /* Note trip through cpu_switchto(). */ 737 pserialize_switchpoint(); 738 739 KASSERT(l->l_cpu == ci); 740 splx(oldspl); 741 /* 742 * note that, unless the caller disabled preemption, 743 * we can be preempted at any time after the above splx() call. 744 */ 745 retval = 1; 746 } else { 747 /* Nothing to do - just unlock and return. */ 748 mutex_spin_exit(spc->spc_mutex); 749 lwp_unlock(l); 750 retval = 0; 751 } 752 753 KASSERT(l == curlwp); 754 KASSERT(l->l_stat == LSONPROC); 755 756 SYSCALL_TIME_WAKEUP(l); 757 LOCKDEBUG_BARRIER(NULL, 1); 758 759 return retval; 760 } 761 762 /* 763 * The machine independent parts of context switch to oblivion. 764 * Does not return. Call with the LWP unlocked. 765 */ 766 void 767 lwp_exit_switchaway(lwp_t *l) 768 { 769 struct cpu_info *ci; 770 struct lwp *newl; 771 struct bintime bt; 772 773 ci = l->l_cpu; 774 775 KASSERT(kpreempt_disabled()); 776 KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL); 777 KASSERT(ci == curcpu()); 778 LOCKDEBUG_BARRIER(NULL, 0); 779 780 kstack_check_magic(l); 781 782 /* Count time spent in current system call */ 783 SYSCALL_TIME_SLEEP(l); 784 binuptime(&bt); 785 updatertime(l, &bt); 786 787 /* Must stay at IPL_SCHED even after releasing run queue lock. */ 788 (void)splsched(); 789 790 /* 791 * Let sched_nextlwp() select the LWP to run the CPU next. 792 * If no LWP is runnable, select the idle LWP. 793 * 794 * Note that spc_lwplock might not necessary be held, and 795 * new thread would be unlocked after setting the LWP-lock. 796 */ 797 spc_lock(ci); 798 #ifndef __HAVE_FAST_SOFTINTS 799 if (ci->ci_data.cpu_softints != 0) { 800 /* There are pending soft interrupts, so pick one. */ 801 newl = softint_picklwp(); 802 newl->l_stat = LSONPROC; 803 newl->l_pflag |= LP_RUNNING; 804 } else 805 #endif /* !__HAVE_FAST_SOFTINTS */ 806 { 807 newl = nextlwp(ci, &ci->ci_schedstate); 808 } 809 810 /* Update the new LWP's start time. */ 811 newl->l_stime = bt; 812 l->l_pflag &= ~LP_RUNNING; 813 814 /* 815 * ci_curlwp changes when a fast soft interrupt occurs. 816 * We use cpu_onproc to keep track of which kernel or 817 * user thread is running 'underneath' the software 818 * interrupt. This is important for time accounting, 819 * itimers and forcing user threads to preempt (aston). 820 */ 821 ci->ci_data.cpu_onproc = newl; 822 823 /* 824 * Preemption related tasks. Must be done with the current 825 * CPU locked. 826 */ 827 cpu_did_resched(l); 828 829 /* Unlock the run queue. */ 830 spc_unlock(ci); 831 832 /* Count the context switch on this CPU. */ 833 ci->ci_data.cpu_nswtch++; 834 835 /* Update status for lwpctl, if present. */ 836 if (l->l_lwpctl != NULL) 837 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 838 839 /* 840 * We may need to spin-wait if 'newl' is still 841 * context switching on another CPU. 842 */ 843 if (__predict_false(newl->l_ctxswtch != 0)) { 844 u_int count; 845 count = SPINLOCK_BACKOFF_MIN; 846 while (newl->l_ctxswtch) 847 SPINLOCK_BACKOFF(count); 848 } 849 850 /* 851 * If DTrace has set the active vtime enum to anything 852 * other than INACTIVE (0), then it should have set the 853 * function to call. 854 */ 855 if (__predict_false(dtrace_vtime_active)) { 856 (*dtrace_vtime_switch_func)(newl); 857 } 858 859 /* Switch to the new LWP.. */ 860 (void)cpu_switchto(NULL, newl, false); 861 862 for (;;) continue; /* XXX: convince gcc about "noreturn" */ 863 /* NOTREACHED */ 864 } 865 866 /* 867 * setrunnable: change LWP state to be runnable, placing it on the run queue. 868 * 869 * Call with the process and LWP locked. Will return with the LWP unlocked. 870 */ 871 void 872 setrunnable(struct lwp *l) 873 { 874 struct proc *p = l->l_proc; 875 struct cpu_info *ci; 876 877 KASSERT((l->l_flag & LW_IDLE) == 0); 878 KASSERT(mutex_owned(p->p_lock)); 879 KASSERT(lwp_locked(l, NULL)); 880 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex); 881 882 switch (l->l_stat) { 883 case LSSTOP: 884 /* 885 * If we're being traced (possibly because someone attached us 886 * while we were stopped), check for a signal from the debugger. 887 */ 888 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xsig != 0) 889 signotify(l); 890 p->p_nrlwps++; 891 break; 892 case LSSUSPENDED: 893 l->l_flag &= ~LW_WSUSPEND; 894 p->p_nrlwps++; 895 cv_broadcast(&p->p_lwpcv); 896 break; 897 case LSSLEEP: 898 KASSERT(l->l_wchan != NULL); 899 break; 900 default: 901 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 902 } 903 904 /* 905 * If the LWP was sleeping, start it again. 906 */ 907 if (l->l_wchan != NULL) { 908 l->l_stat = LSSLEEP; 909 /* lwp_unsleep() will release the lock. */ 910 lwp_unsleep(l, true); 911 return; 912 } 913 914 /* 915 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 916 * about to call mi_switch(), in which case it will yield. 917 */ 918 if ((l->l_pflag & LP_RUNNING) != 0) { 919 l->l_stat = LSONPROC; 920 l->l_slptime = 0; 921 lwp_unlock(l); 922 return; 923 } 924 925 /* 926 * Look for a CPU to run. 927 * Set the LWP runnable. 928 */ 929 ci = sched_takecpu(l); 930 l->l_cpu = ci; 931 spc_lock(ci); 932 lwp_unlock_to(l, ci->ci_schedstate.spc_mutex); 933 sched_setrunnable(l); 934 l->l_stat = LSRUN; 935 l->l_slptime = 0; 936 937 sched_enqueue(l, false); 938 resched_cpu(l); 939 lwp_unlock(l); 940 } 941 942 /* 943 * suspendsched: 944 * 945 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 946 */ 947 void 948 suspendsched(void) 949 { 950 CPU_INFO_ITERATOR cii; 951 struct cpu_info *ci; 952 struct lwp *l; 953 struct proc *p; 954 955 /* 956 * We do this by process in order not to violate the locking rules. 957 */ 958 mutex_enter(proc_lock); 959 PROCLIST_FOREACH(p, &allproc) { 960 mutex_enter(p->p_lock); 961 if ((p->p_flag & PK_SYSTEM) != 0) { 962 mutex_exit(p->p_lock); 963 continue; 964 } 965 966 if (p->p_stat != SSTOP) { 967 if (p->p_stat != SZOMB && p->p_stat != SDEAD) { 968 p->p_pptr->p_nstopchild++; 969 p->p_waited = 0; 970 } 971 p->p_stat = SSTOP; 972 } 973 974 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 975 if (l == curlwp) 976 continue; 977 978 lwp_lock(l); 979 980 /* 981 * Set L_WREBOOT so that the LWP will suspend itself 982 * when it tries to return to user mode. We want to 983 * try and get to get as many LWPs as possible to 984 * the user / kernel boundary, so that they will 985 * release any locks that they hold. 986 */ 987 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 988 989 if (l->l_stat == LSSLEEP && 990 (l->l_flag & LW_SINTR) != 0) { 991 /* setrunnable() will release the lock. */ 992 setrunnable(l); 993 continue; 994 } 995 996 lwp_unlock(l); 997 } 998 999 mutex_exit(p->p_lock); 1000 } 1001 mutex_exit(proc_lock); 1002 1003 /* 1004 * Kick all CPUs to make them preempt any LWPs running in user mode. 1005 * They'll trap into the kernel and suspend themselves in userret(). 1006 */ 1007 for (CPU_INFO_FOREACH(cii, ci)) { 1008 spc_lock(ci); 1009 cpu_need_resched(ci, RESCHED_IMMED); 1010 spc_unlock(ci); 1011 } 1012 } 1013 1014 /* 1015 * sched_unsleep: 1016 * 1017 * The is called when the LWP has not been awoken normally but instead 1018 * interrupted: for example, if the sleep timed out. Because of this, 1019 * it's not a valid action for running or idle LWPs. 1020 */ 1021 static void 1022 sched_unsleep(struct lwp *l, bool cleanup) 1023 { 1024 1025 lwp_unlock(l); 1026 panic("sched_unsleep"); 1027 } 1028 1029 static void 1030 resched_cpu(struct lwp *l) 1031 { 1032 struct cpu_info *ci = l->l_cpu; 1033 1034 KASSERT(lwp_locked(l, NULL)); 1035 if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority) 1036 cpu_need_resched(ci, 0); 1037 } 1038 1039 static void 1040 sched_changepri(struct lwp *l, pri_t pri) 1041 { 1042 1043 KASSERT(lwp_locked(l, NULL)); 1044 1045 if (l->l_stat == LSRUN) { 1046 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1047 sched_dequeue(l); 1048 l->l_priority = pri; 1049 sched_enqueue(l, false); 1050 } else { 1051 l->l_priority = pri; 1052 } 1053 resched_cpu(l); 1054 } 1055 1056 static void 1057 sched_lendpri(struct lwp *l, pri_t pri) 1058 { 1059 1060 KASSERT(lwp_locked(l, NULL)); 1061 1062 if (l->l_stat == LSRUN) { 1063 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1064 sched_dequeue(l); 1065 l->l_inheritedprio = pri; 1066 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1067 sched_enqueue(l, false); 1068 } else { 1069 l->l_inheritedprio = pri; 1070 l->l_auxprio = MAX(l->l_inheritedprio, l->l_protectprio); 1071 } 1072 resched_cpu(l); 1073 } 1074 1075 struct lwp * 1076 syncobj_noowner(wchan_t wchan) 1077 { 1078 1079 return NULL; 1080 } 1081 1082 /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */ 1083 const fixpt_t ccpu = 0.95122942450071400909 * FSCALE; 1084 1085 /* 1086 * Constants for averages over 1, 5 and 15 minutes when sampling at 1087 * 5 second intervals. 1088 */ 1089 static const fixpt_t cexp[ ] = { 1090 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 1091 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 1092 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 1093 }; 1094 1095 /* 1096 * sched_pstats: 1097 * 1098 * => Update process statistics and check CPU resource allocation. 1099 * => Call scheduler-specific hook to eventually adjust LWP priorities. 1100 * => Compute load average of a quantity on 1, 5 and 15 minute intervals. 1101 */ 1102 void 1103 sched_pstats(void) 1104 { 1105 extern struct loadavg averunnable; 1106 struct loadavg *avg = &averunnable; 1107 const int clkhz = (stathz != 0 ? stathz : hz); 1108 static bool backwards = false; 1109 static u_int lavg_count = 0; 1110 struct proc *p; 1111 int nrun; 1112 1113 sched_pstats_ticks++; 1114 if (++lavg_count >= 5) { 1115 lavg_count = 0; 1116 nrun = 0; 1117 } 1118 mutex_enter(proc_lock); 1119 PROCLIST_FOREACH(p, &allproc) { 1120 struct lwp *l; 1121 struct rlimit *rlim; 1122 time_t runtm; 1123 int sig; 1124 1125 /* Increment sleep time (if sleeping), ignore overflow. */ 1126 mutex_enter(p->p_lock); 1127 runtm = p->p_rtime.sec; 1128 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1129 fixpt_t lpctcpu; 1130 u_int lcpticks; 1131 1132 if (__predict_false((l->l_flag & LW_IDLE) != 0)) 1133 continue; 1134 lwp_lock(l); 1135 runtm += l->l_rtime.sec; 1136 l->l_swtime++; 1137 sched_lwp_stats(l); 1138 1139 /* For load average calculation. */ 1140 if (__predict_false(lavg_count == 0) && 1141 (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) { 1142 switch (l->l_stat) { 1143 case LSSLEEP: 1144 if (l->l_slptime > 1) { 1145 break; 1146 } 1147 case LSRUN: 1148 case LSONPROC: 1149 case LSIDL: 1150 nrun++; 1151 } 1152 } 1153 lwp_unlock(l); 1154 1155 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 1156 if (l->l_slptime != 0) 1157 continue; 1158 1159 lpctcpu = l->l_pctcpu; 1160 lcpticks = atomic_swap_uint(&l->l_cpticks, 0); 1161 lpctcpu += ((FSCALE - ccpu) * 1162 (lcpticks * FSCALE / clkhz)) >> FSHIFT; 1163 l->l_pctcpu = lpctcpu; 1164 } 1165 /* Calculating p_pctcpu only for ps(1) */ 1166 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 1167 1168 if (__predict_false(runtm < 0)) { 1169 if (!backwards) { 1170 backwards = true; 1171 printf("WARNING: negative runtime; " 1172 "monotonic clock has gone backwards\n"); 1173 } 1174 mutex_exit(p->p_lock); 1175 continue; 1176 } 1177 1178 /* 1179 * Check if the process exceeds its CPU resource allocation. 1180 * If over the hard limit, kill it with SIGKILL. 1181 * If over the soft limit, send SIGXCPU and raise 1182 * the soft limit a little. 1183 */ 1184 rlim = &p->p_rlimit[RLIMIT_CPU]; 1185 sig = 0; 1186 if (__predict_false(runtm >= rlim->rlim_cur)) { 1187 if (runtm >= rlim->rlim_max) { 1188 sig = SIGKILL; 1189 log(LOG_NOTICE, 1190 "pid %d, command %s, is killed: %s\n", 1191 p->p_pid, p->p_comm, "exceeded RLIMIT_CPU"); 1192 uprintf("pid %d, command %s, is killed: %s\n", 1193 p->p_pid, p->p_comm, "exceeded RLIMIT_CPU"); 1194 } else { 1195 sig = SIGXCPU; 1196 if (rlim->rlim_cur < rlim->rlim_max) 1197 rlim->rlim_cur += 5; 1198 } 1199 } 1200 mutex_exit(p->p_lock); 1201 if (__predict_false(sig)) { 1202 KASSERT((p->p_flag & PK_SYSTEM) == 0); 1203 psignal(p, sig); 1204 } 1205 } 1206 mutex_exit(proc_lock); 1207 1208 /* Load average calculation. */ 1209 if (__predict_false(lavg_count == 0)) { 1210 int i; 1211 CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg)); 1212 for (i = 0; i < __arraycount(cexp); i++) { 1213 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 1214 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 1215 } 1216 } 1217 1218 /* Lightning bolt. */ 1219 cv_broadcast(&lbolt); 1220 } 1221