1 /* $NetBSD: kern_synch.c,v 1.309 2015/10/13 00:25:51 pgoyette Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 11 * Daniel Sieger. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1990, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.309 2015/10/13 00:25:51 pgoyette Exp $"); 73 74 #include "opt_kstack.h" 75 #include "opt_perfctrs.h" 76 #include "opt_dtrace.h" 77 78 #define __MUTEX_PRIVATE 79 80 #include <sys/param.h> 81 #include <sys/systm.h> 82 #include <sys/proc.h> 83 #include <sys/kernel.h> 84 #if defined(PERFCTRS) 85 #include <sys/pmc.h> 86 #endif 87 #include <sys/cpu.h> 88 #include <sys/pserialize.h> 89 #include <sys/resourcevar.h> 90 #include <sys/sched.h> 91 #include <sys/syscall_stats.h> 92 #include <sys/sleepq.h> 93 #include <sys/lockdebug.h> 94 #include <sys/evcnt.h> 95 #include <sys/intr.h> 96 #include <sys/lwpctl.h> 97 #include <sys/atomic.h> 98 #include <sys/syslog.h> 99 100 #include <uvm/uvm_extern.h> 101 102 #include <dev/lockstat.h> 103 104 #include <sys/dtrace_bsd.h> 105 int dtrace_vtime_active=0; 106 dtrace_vtime_switch_func_t dtrace_vtime_switch_func; 107 108 static void sched_unsleep(struct lwp *, bool); 109 static void sched_changepri(struct lwp *, pri_t); 110 static void sched_lendpri(struct lwp *, pri_t); 111 static void resched_cpu(struct lwp *); 112 113 syncobj_t sleep_syncobj = { 114 SOBJ_SLEEPQ_SORTED, 115 sleepq_unsleep, 116 sleepq_changepri, 117 sleepq_lendpri, 118 syncobj_noowner, 119 }; 120 121 syncobj_t sched_syncobj = { 122 SOBJ_SLEEPQ_SORTED, 123 sched_unsleep, 124 sched_changepri, 125 sched_lendpri, 126 syncobj_noowner, 127 }; 128 129 /* "Lightning bolt": once a second sleep address. */ 130 kcondvar_t lbolt __cacheline_aligned; 131 132 u_int sched_pstats_ticks __cacheline_aligned; 133 134 /* Preemption event counters. */ 135 static struct evcnt kpreempt_ev_crit __cacheline_aligned; 136 static struct evcnt kpreempt_ev_klock __cacheline_aligned; 137 static struct evcnt kpreempt_ev_immed __cacheline_aligned; 138 139 void 140 synch_init(void) 141 { 142 143 cv_init(&lbolt, "lbolt"); 144 145 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL, 146 "kpreempt", "defer: critical section"); 147 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL, 148 "kpreempt", "defer: kernel_lock"); 149 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL, 150 "kpreempt", "immediate"); 151 } 152 153 /* 154 * OBSOLETE INTERFACE 155 * 156 * General sleep call. Suspends the current LWP until a wakeup is 157 * performed on the specified identifier. The LWP will then be made 158 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 159 * means no timeout). If pri includes PCATCH flag, signals are checked 160 * before and after sleeping, else signals are not checked. Returns 0 if 161 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 162 * signal needs to be delivered, ERESTART is returned if the current system 163 * call should be restarted if possible, and EINTR is returned if the system 164 * call should be interrupted by the signal (return EINTR). 165 */ 166 int 167 tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo) 168 { 169 struct lwp *l = curlwp; 170 sleepq_t *sq; 171 kmutex_t *mp; 172 173 KASSERT((l->l_pflag & LP_INTR) == 0); 174 KASSERT(ident != &lbolt); 175 176 if (sleepq_dontsleep(l)) { 177 (void)sleepq_abort(NULL, 0); 178 return 0; 179 } 180 181 l->l_kpriority = true; 182 sq = sleeptab_lookup(&sleeptab, ident, &mp); 183 sleepq_enter(sq, l, mp); 184 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 185 return sleepq_block(timo, priority & PCATCH); 186 } 187 188 int 189 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 190 kmutex_t *mtx) 191 { 192 struct lwp *l = curlwp; 193 sleepq_t *sq; 194 kmutex_t *mp; 195 int error; 196 197 KASSERT((l->l_pflag & LP_INTR) == 0); 198 KASSERT(ident != &lbolt); 199 200 if (sleepq_dontsleep(l)) { 201 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 202 return 0; 203 } 204 205 l->l_kpriority = true; 206 sq = sleeptab_lookup(&sleeptab, ident, &mp); 207 sleepq_enter(sq, l, mp); 208 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 209 mutex_exit(mtx); 210 error = sleepq_block(timo, priority & PCATCH); 211 212 if ((priority & PNORELOCK) == 0) 213 mutex_enter(mtx); 214 215 return error; 216 } 217 218 /* 219 * General sleep call for situations where a wake-up is not expected. 220 */ 221 int 222 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 223 { 224 struct lwp *l = curlwp; 225 kmutex_t *mp; 226 sleepq_t *sq; 227 int error; 228 229 KASSERT(!(timo == 0 && intr == false)); 230 231 if (sleepq_dontsleep(l)) 232 return sleepq_abort(NULL, 0); 233 234 if (mtx != NULL) 235 mutex_exit(mtx); 236 l->l_kpriority = true; 237 sq = sleeptab_lookup(&sleeptab, l, &mp); 238 sleepq_enter(sq, l, mp); 239 sleepq_enqueue(sq, l, wmesg, &sleep_syncobj); 240 error = sleepq_block(timo, intr); 241 if (mtx != NULL) 242 mutex_enter(mtx); 243 244 return error; 245 } 246 247 /* 248 * OBSOLETE INTERFACE 249 * 250 * Make all LWPs sleeping on the specified identifier runnable. 251 */ 252 void 253 wakeup(wchan_t ident) 254 { 255 sleepq_t *sq; 256 kmutex_t *mp; 257 258 if (__predict_false(cold)) 259 return; 260 261 sq = sleeptab_lookup(&sleeptab, ident, &mp); 262 sleepq_wake(sq, ident, (u_int)-1, mp); 263 } 264 265 /* 266 * General yield call. Puts the current LWP back on its run queue and 267 * performs a voluntary context switch. Should only be called when the 268 * current LWP explicitly requests it (eg sched_yield(2)). 269 */ 270 void 271 yield(void) 272 { 273 struct lwp *l = curlwp; 274 275 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 276 lwp_lock(l); 277 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 278 KASSERT(l->l_stat == LSONPROC); 279 l->l_kpriority = false; 280 (void)mi_switch(l); 281 KERNEL_LOCK(l->l_biglocks, l); 282 } 283 284 /* 285 * General preemption call. Puts the current LWP back on its run queue 286 * and performs an involuntary context switch. 287 */ 288 void 289 preempt(void) 290 { 291 struct lwp *l = curlwp; 292 293 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 294 lwp_lock(l); 295 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 296 KASSERT(l->l_stat == LSONPROC); 297 l->l_kpriority = false; 298 l->l_nivcsw++; 299 (void)mi_switch(l); 300 KERNEL_LOCK(l->l_biglocks, l); 301 } 302 303 /* 304 * Handle a request made by another agent to preempt the current LWP 305 * in-kernel. Usually called when l_dopreempt may be non-zero. 306 * 307 * Character addresses for lockstat only. 308 */ 309 static char in_critical_section; 310 static char kernel_lock_held; 311 static char is_softint; 312 static char cpu_kpreempt_enter_fail; 313 314 bool 315 kpreempt(uintptr_t where) 316 { 317 uintptr_t failed; 318 lwp_t *l; 319 int s, dop, lsflag; 320 321 l = curlwp; 322 failed = 0; 323 while ((dop = l->l_dopreempt) != 0) { 324 if (l->l_stat != LSONPROC) { 325 /* 326 * About to block (or die), let it happen. 327 * Doesn't really count as "preemption has 328 * been blocked", since we're going to 329 * context switch. 330 */ 331 l->l_dopreempt = 0; 332 return true; 333 } 334 if (__predict_false((l->l_flag & LW_IDLE) != 0)) { 335 /* Can't preempt idle loop, don't count as failure. */ 336 l->l_dopreempt = 0; 337 return true; 338 } 339 if (__predict_false(l->l_nopreempt != 0)) { 340 /* LWP holds preemption disabled, explicitly. */ 341 if ((dop & DOPREEMPT_COUNTED) == 0) { 342 kpreempt_ev_crit.ev_count++; 343 } 344 failed = (uintptr_t)&in_critical_section; 345 break; 346 } 347 if (__predict_false((l->l_pflag & LP_INTR) != 0)) { 348 /* Can't preempt soft interrupts yet. */ 349 l->l_dopreempt = 0; 350 failed = (uintptr_t)&is_softint; 351 break; 352 } 353 s = splsched(); 354 if (__predict_false(l->l_blcnt != 0 || 355 curcpu()->ci_biglock_wanted != NULL)) { 356 /* Hold or want kernel_lock, code is not MT safe. */ 357 splx(s); 358 if ((dop & DOPREEMPT_COUNTED) == 0) { 359 kpreempt_ev_klock.ev_count++; 360 } 361 failed = (uintptr_t)&kernel_lock_held; 362 break; 363 } 364 if (__predict_false(!cpu_kpreempt_enter(where, s))) { 365 /* 366 * It may be that the IPL is too high. 367 * kpreempt_enter() can schedule an 368 * interrupt to retry later. 369 */ 370 splx(s); 371 failed = (uintptr_t)&cpu_kpreempt_enter_fail; 372 break; 373 } 374 /* Do it! */ 375 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) { 376 kpreempt_ev_immed.ev_count++; 377 } 378 lwp_lock(l); 379 mi_switch(l); 380 l->l_nopreempt++; 381 splx(s); 382 383 /* Take care of any MD cleanup. */ 384 cpu_kpreempt_exit(where); 385 l->l_nopreempt--; 386 } 387 388 if (__predict_true(!failed)) { 389 return false; 390 } 391 392 /* Record preemption failure for reporting via lockstat. */ 393 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED); 394 lsflag = 0; 395 LOCKSTAT_ENTER(lsflag); 396 if (__predict_false(lsflag)) { 397 if (where == 0) { 398 where = (uintptr_t)__builtin_return_address(0); 399 } 400 /* Preemption is on, might recurse, so make it atomic. */ 401 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL, 402 (void *)where) == NULL) { 403 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime); 404 l->l_pfaillock = failed; 405 } 406 } 407 LOCKSTAT_EXIT(lsflag); 408 return true; 409 } 410 411 /* 412 * Return true if preemption is explicitly disabled. 413 */ 414 bool 415 kpreempt_disabled(void) 416 { 417 const lwp_t *l = curlwp; 418 419 return l->l_nopreempt != 0 || l->l_stat == LSZOMB || 420 (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled(); 421 } 422 423 /* 424 * Disable kernel preemption. 425 */ 426 void 427 kpreempt_disable(void) 428 { 429 430 KPREEMPT_DISABLE(curlwp); 431 } 432 433 /* 434 * Reenable kernel preemption. 435 */ 436 void 437 kpreempt_enable(void) 438 { 439 440 KPREEMPT_ENABLE(curlwp); 441 } 442 443 /* 444 * Compute the amount of time during which the current lwp was running. 445 * 446 * - update l_rtime unless it's an idle lwp. 447 */ 448 449 void 450 updatertime(lwp_t *l, const struct bintime *now) 451 { 452 453 if (__predict_false(l->l_flag & LW_IDLE)) 454 return; 455 456 /* rtime += now - stime */ 457 bintime_add(&l->l_rtime, now); 458 bintime_sub(&l->l_rtime, &l->l_stime); 459 } 460 461 /* 462 * Select next LWP from the current CPU to run.. 463 */ 464 static inline lwp_t * 465 nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc) 466 { 467 lwp_t *newl; 468 469 /* 470 * Let sched_nextlwp() select the LWP to run the CPU next. 471 * If no LWP is runnable, select the idle LWP. 472 * 473 * Note that spc_lwplock might not necessary be held, and 474 * new thread would be unlocked after setting the LWP-lock. 475 */ 476 newl = sched_nextlwp(); 477 if (newl != NULL) { 478 sched_dequeue(newl); 479 KASSERT(lwp_locked(newl, spc->spc_mutex)); 480 KASSERT(newl->l_cpu == ci); 481 newl->l_stat = LSONPROC; 482 newl->l_pflag |= LP_RUNNING; 483 lwp_setlock(newl, spc->spc_lwplock); 484 } else { 485 newl = ci->ci_data.cpu_idlelwp; 486 newl->l_stat = LSONPROC; 487 newl->l_pflag |= LP_RUNNING; 488 } 489 490 /* 491 * Only clear want_resched if there are no pending (slow) 492 * software interrupts. 493 */ 494 ci->ci_want_resched = ci->ci_data.cpu_softints; 495 spc->spc_flags &= ~SPCF_SWITCHCLEAR; 496 spc->spc_curpriority = lwp_eprio(newl); 497 498 return newl; 499 } 500 501 /* 502 * The machine independent parts of context switch. 503 * 504 * Returns 1 if another LWP was actually run. 505 */ 506 int 507 mi_switch(lwp_t *l) 508 { 509 struct cpu_info *ci; 510 struct schedstate_percpu *spc; 511 struct lwp *newl; 512 int retval, oldspl; 513 struct bintime bt; 514 bool returning; 515 516 KASSERT(lwp_locked(l, NULL)); 517 KASSERT(kpreempt_disabled()); 518 LOCKDEBUG_BARRIER(l->l_mutex, 1); 519 520 kstack_check_magic(l); 521 522 binuptime(&bt); 523 524 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 525 KASSERT((l->l_pflag & LP_RUNNING) != 0); 526 KASSERT(l->l_cpu == curcpu()); 527 ci = l->l_cpu; 528 spc = &ci->ci_schedstate; 529 returning = false; 530 newl = NULL; 531 532 /* 533 * If we have been asked to switch to a specific LWP, then there 534 * is no need to inspect the run queues. If a soft interrupt is 535 * blocking, then return to the interrupted thread without adjusting 536 * VM context or its start time: neither have been changed in order 537 * to take the interrupt. 538 */ 539 if (l->l_switchto != NULL) { 540 if ((l->l_pflag & LP_INTR) != 0) { 541 returning = true; 542 softint_block(l); 543 if ((l->l_pflag & LP_TIMEINTR) != 0) 544 updatertime(l, &bt); 545 } 546 newl = l->l_switchto; 547 l->l_switchto = NULL; 548 } 549 #ifndef __HAVE_FAST_SOFTINTS 550 else if (ci->ci_data.cpu_softints != 0) { 551 /* There are pending soft interrupts, so pick one. */ 552 newl = softint_picklwp(); 553 newl->l_stat = LSONPROC; 554 newl->l_pflag |= LP_RUNNING; 555 } 556 #endif /* !__HAVE_FAST_SOFTINTS */ 557 558 /* Count time spent in current system call */ 559 if (!returning) { 560 SYSCALL_TIME_SLEEP(l); 561 562 /* 563 * XXXSMP If we are using h/w performance counters, 564 * save context. 565 */ 566 #if PERFCTRS 567 if (PMC_ENABLED(l->l_proc)) { 568 pmc_save_context(l->l_proc); 569 } 570 #endif 571 updatertime(l, &bt); 572 } 573 574 /* Lock the runqueue */ 575 KASSERT(l->l_stat != LSRUN); 576 mutex_spin_enter(spc->spc_mutex); 577 578 /* 579 * If on the CPU and we have gotten this far, then we must yield. 580 */ 581 if (l->l_stat == LSONPROC && l != newl) { 582 KASSERT(lwp_locked(l, spc->spc_lwplock)); 583 if ((l->l_flag & LW_IDLE) == 0) { 584 l->l_stat = LSRUN; 585 lwp_setlock(l, spc->spc_mutex); 586 sched_enqueue(l, true); 587 /* 588 * Handle migration. Note that "migrating LWP" may 589 * be reset here, if interrupt/preemption happens 590 * early in idle LWP. 591 */ 592 if (l->l_target_cpu != NULL) { 593 KASSERT((l->l_pflag & LP_INTR) == 0); 594 spc->spc_migrating = l; 595 } 596 } else 597 l->l_stat = LSIDL; 598 } 599 600 /* Pick new LWP to run. */ 601 if (newl == NULL) { 602 newl = nextlwp(ci, spc); 603 } 604 605 /* Items that must be updated with the CPU locked. */ 606 if (!returning) { 607 /* Update the new LWP's start time. */ 608 newl->l_stime = bt; 609 610 /* 611 * ci_curlwp changes when a fast soft interrupt occurs. 612 * We use cpu_onproc to keep track of which kernel or 613 * user thread is running 'underneath' the software 614 * interrupt. This is important for time accounting, 615 * itimers and forcing user threads to preempt (aston). 616 */ 617 ci->ci_data.cpu_onproc = newl; 618 } 619 620 /* 621 * Preemption related tasks. Must be done with the current 622 * CPU locked. 623 */ 624 cpu_did_resched(l); 625 l->l_dopreempt = 0; 626 if (__predict_false(l->l_pfailaddr != 0)) { 627 LOCKSTAT_FLAG(lsflag); 628 LOCKSTAT_ENTER(lsflag); 629 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime); 630 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN, 631 1, l->l_pfailtime, l->l_pfailaddr); 632 LOCKSTAT_EXIT(lsflag); 633 l->l_pfailtime = 0; 634 l->l_pfaillock = 0; 635 l->l_pfailaddr = 0; 636 } 637 638 if (l != newl) { 639 struct lwp *prevlwp; 640 641 /* Release all locks, but leave the current LWP locked */ 642 if (l->l_mutex == spc->spc_mutex) { 643 /* 644 * Drop spc_lwplock, if the current LWP has been moved 645 * to the run queue (it is now locked by spc_mutex). 646 */ 647 mutex_spin_exit(spc->spc_lwplock); 648 } else { 649 /* 650 * Otherwise, drop the spc_mutex, we are done with the 651 * run queues. 652 */ 653 mutex_spin_exit(spc->spc_mutex); 654 } 655 656 /* 657 * Mark that context switch is going to be performed 658 * for this LWP, to protect it from being switched 659 * to on another CPU. 660 */ 661 KASSERT(l->l_ctxswtch == 0); 662 l->l_ctxswtch = 1; 663 l->l_ncsw++; 664 KASSERT((l->l_pflag & LP_RUNNING) != 0); 665 l->l_pflag &= ~LP_RUNNING; 666 667 /* 668 * Increase the count of spin-mutexes before the release 669 * of the last lock - we must remain at IPL_SCHED during 670 * the context switch. 671 */ 672 KASSERTMSG(ci->ci_mtx_count == -1, 673 "%s: cpu%u: ci_mtx_count (%d) != -1 " 674 "(block with spin-mutex held)", 675 __func__, cpu_index(ci), ci->ci_mtx_count); 676 oldspl = MUTEX_SPIN_OLDSPL(ci); 677 ci->ci_mtx_count--; 678 lwp_unlock(l); 679 680 /* Count the context switch on this CPU. */ 681 ci->ci_data.cpu_nswtch++; 682 683 /* Update status for lwpctl, if present. */ 684 if (l->l_lwpctl != NULL) 685 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE; 686 687 /* 688 * Save old VM context, unless a soft interrupt 689 * handler is blocking. 690 */ 691 if (!returning) 692 pmap_deactivate(l); 693 694 /* 695 * We may need to spin-wait if 'newl' is still 696 * context switching on another CPU. 697 */ 698 if (__predict_false(newl->l_ctxswtch != 0)) { 699 u_int count; 700 count = SPINLOCK_BACKOFF_MIN; 701 while (newl->l_ctxswtch) 702 SPINLOCK_BACKOFF(count); 703 } 704 705 /* 706 * If DTrace has set the active vtime enum to anything 707 * other than INACTIVE (0), then it should have set the 708 * function to call. 709 */ 710 if (__predict_false(dtrace_vtime_active)) { 711 (*dtrace_vtime_switch_func)(newl); 712 } 713 714 /* Switch to the new LWP.. */ 715 #ifdef MULTIPROCESSOR 716 KASSERT(curlwp == ci->ci_curlwp); 717 #endif 718 KASSERTMSG(l == curlwp, "l %p curlwp %p", l, curlwp); 719 prevlwp = cpu_switchto(l, newl, returning); 720 ci = curcpu(); 721 #ifdef MULTIPROCESSOR 722 KASSERT(curlwp == ci->ci_curlwp); 723 #endif 724 KASSERTMSG(l == curlwp, "l %p curlwp %p prevlwp %p", 725 l, curlwp, prevlwp); 726 727 /* 728 * Switched away - we have new curlwp. 729 * Restore VM context and IPL. 730 */ 731 pmap_activate(l); 732 uvm_emap_switch(l); 733 pcu_switchpoint(l); 734 735 if (prevlwp != NULL) { 736 /* Normalize the count of the spin-mutexes */ 737 ci->ci_mtx_count++; 738 /* Unmark the state of context switch */ 739 membar_exit(); 740 prevlwp->l_ctxswtch = 0; 741 } 742 743 /* Update status for lwpctl, if present. */ 744 if (l->l_lwpctl != NULL) { 745 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); 746 l->l_lwpctl->lc_pctr++; 747 } 748 749 /* Note trip through cpu_switchto(). */ 750 pserialize_switchpoint(); 751 752 KASSERT(l->l_cpu == ci); 753 splx(oldspl); 754 /* 755 * note that, unless the caller disabled preemption, 756 * we can be preempted at any time after the above splx() call. 757 */ 758 retval = 1; 759 } else { 760 /* Nothing to do - just unlock and return. */ 761 mutex_spin_exit(spc->spc_mutex); 762 lwp_unlock(l); 763 retval = 0; 764 } 765 766 KASSERT(l == curlwp); 767 KASSERT(l->l_stat == LSONPROC); 768 769 /* 770 * XXXSMP If we are using h/w performance counters, restore context. 771 * XXXSMP preemption problem. 772 */ 773 #if PERFCTRS 774 if (PMC_ENABLED(l->l_proc)) { 775 pmc_restore_context(l->l_proc); 776 } 777 #endif 778 SYSCALL_TIME_WAKEUP(l); 779 LOCKDEBUG_BARRIER(NULL, 1); 780 781 return retval; 782 } 783 784 /* 785 * The machine independent parts of context switch to oblivion. 786 * Does not return. Call with the LWP unlocked. 787 */ 788 void 789 lwp_exit_switchaway(lwp_t *l) 790 { 791 struct cpu_info *ci; 792 struct lwp *newl; 793 struct bintime bt; 794 795 ci = l->l_cpu; 796 797 KASSERT(kpreempt_disabled()); 798 KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL); 799 KASSERT(ci == curcpu()); 800 LOCKDEBUG_BARRIER(NULL, 0); 801 802 kstack_check_magic(l); 803 804 /* Count time spent in current system call */ 805 SYSCALL_TIME_SLEEP(l); 806 binuptime(&bt); 807 updatertime(l, &bt); 808 809 /* Must stay at IPL_SCHED even after releasing run queue lock. */ 810 (void)splsched(); 811 812 /* 813 * Let sched_nextlwp() select the LWP to run the CPU next. 814 * If no LWP is runnable, select the idle LWP. 815 * 816 * Note that spc_lwplock might not necessary be held, and 817 * new thread would be unlocked after setting the LWP-lock. 818 */ 819 spc_lock(ci); 820 #ifndef __HAVE_FAST_SOFTINTS 821 if (ci->ci_data.cpu_softints != 0) { 822 /* There are pending soft interrupts, so pick one. */ 823 newl = softint_picklwp(); 824 newl->l_stat = LSONPROC; 825 newl->l_pflag |= LP_RUNNING; 826 } else 827 #endif /* !__HAVE_FAST_SOFTINTS */ 828 { 829 newl = nextlwp(ci, &ci->ci_schedstate); 830 } 831 832 /* Update the new LWP's start time. */ 833 newl->l_stime = bt; 834 l->l_pflag &= ~LP_RUNNING; 835 836 /* 837 * ci_curlwp changes when a fast soft interrupt occurs. 838 * We use cpu_onproc to keep track of which kernel or 839 * user thread is running 'underneath' the software 840 * interrupt. This is important for time accounting, 841 * itimers and forcing user threads to preempt (aston). 842 */ 843 ci->ci_data.cpu_onproc = newl; 844 845 /* 846 * Preemption related tasks. Must be done with the current 847 * CPU locked. 848 */ 849 cpu_did_resched(l); 850 851 /* Unlock the run queue. */ 852 spc_unlock(ci); 853 854 /* Count the context switch on this CPU. */ 855 ci->ci_data.cpu_nswtch++; 856 857 /* Update status for lwpctl, if present. */ 858 if (l->l_lwpctl != NULL) 859 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 860 861 /* 862 * We may need to spin-wait if 'newl' is still 863 * context switching on another CPU. 864 */ 865 if (__predict_false(newl->l_ctxswtch != 0)) { 866 u_int count; 867 count = SPINLOCK_BACKOFF_MIN; 868 while (newl->l_ctxswtch) 869 SPINLOCK_BACKOFF(count); 870 } 871 872 /* 873 * If DTrace has set the active vtime enum to anything 874 * other than INACTIVE (0), then it should have set the 875 * function to call. 876 */ 877 if (__predict_false(dtrace_vtime_active)) { 878 (*dtrace_vtime_switch_func)(newl); 879 } 880 881 /* Switch to the new LWP.. */ 882 (void)cpu_switchto(NULL, newl, false); 883 884 for (;;) continue; /* XXX: convince gcc about "noreturn" */ 885 /* NOTREACHED */ 886 } 887 888 /* 889 * setrunnable: change LWP state to be runnable, placing it on the run queue. 890 * 891 * Call with the process and LWP locked. Will return with the LWP unlocked. 892 */ 893 void 894 setrunnable(struct lwp *l) 895 { 896 struct proc *p = l->l_proc; 897 struct cpu_info *ci; 898 899 KASSERT((l->l_flag & LW_IDLE) == 0); 900 KASSERT(mutex_owned(p->p_lock)); 901 KASSERT(lwp_locked(l, NULL)); 902 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex); 903 904 switch (l->l_stat) { 905 case LSSTOP: 906 /* 907 * If we're being traced (possibly because someone attached us 908 * while we were stopped), check for a signal from the debugger. 909 */ 910 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0) 911 signotify(l); 912 p->p_nrlwps++; 913 break; 914 case LSSUSPENDED: 915 l->l_flag &= ~LW_WSUSPEND; 916 p->p_nrlwps++; 917 cv_broadcast(&p->p_lwpcv); 918 break; 919 case LSSLEEP: 920 KASSERT(l->l_wchan != NULL); 921 break; 922 default: 923 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 924 } 925 926 /* 927 * If the LWP was sleeping, start it again. 928 */ 929 if (l->l_wchan != NULL) { 930 l->l_stat = LSSLEEP; 931 /* lwp_unsleep() will release the lock. */ 932 lwp_unsleep(l, true); 933 return; 934 } 935 936 /* 937 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 938 * about to call mi_switch(), in which case it will yield. 939 */ 940 if ((l->l_pflag & LP_RUNNING) != 0) { 941 l->l_stat = LSONPROC; 942 l->l_slptime = 0; 943 lwp_unlock(l); 944 return; 945 } 946 947 /* 948 * Look for a CPU to run. 949 * Set the LWP runnable. 950 */ 951 ci = sched_takecpu(l); 952 l->l_cpu = ci; 953 spc_lock(ci); 954 lwp_unlock_to(l, ci->ci_schedstate.spc_mutex); 955 sched_setrunnable(l); 956 l->l_stat = LSRUN; 957 l->l_slptime = 0; 958 959 sched_enqueue(l, false); 960 resched_cpu(l); 961 lwp_unlock(l); 962 } 963 964 /* 965 * suspendsched: 966 * 967 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 968 */ 969 void 970 suspendsched(void) 971 { 972 CPU_INFO_ITERATOR cii; 973 struct cpu_info *ci; 974 struct lwp *l; 975 struct proc *p; 976 977 /* 978 * We do this by process in order not to violate the locking rules. 979 */ 980 mutex_enter(proc_lock); 981 PROCLIST_FOREACH(p, &allproc) { 982 mutex_enter(p->p_lock); 983 if ((p->p_flag & PK_SYSTEM) != 0) { 984 mutex_exit(p->p_lock); 985 continue; 986 } 987 988 if (p->p_stat != SSTOP) { 989 if (p->p_stat != SZOMB && p->p_stat != SDEAD) { 990 p->p_pptr->p_nstopchild++; 991 p->p_waited = 0; 992 } 993 p->p_stat = SSTOP; 994 } 995 996 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 997 if (l == curlwp) 998 continue; 999 1000 lwp_lock(l); 1001 1002 /* 1003 * Set L_WREBOOT so that the LWP will suspend itself 1004 * when it tries to return to user mode. We want to 1005 * try and get to get as many LWPs as possible to 1006 * the user / kernel boundary, so that they will 1007 * release any locks that they hold. 1008 */ 1009 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 1010 1011 if (l->l_stat == LSSLEEP && 1012 (l->l_flag & LW_SINTR) != 0) { 1013 /* setrunnable() will release the lock. */ 1014 setrunnable(l); 1015 continue; 1016 } 1017 1018 lwp_unlock(l); 1019 } 1020 1021 mutex_exit(p->p_lock); 1022 } 1023 mutex_exit(proc_lock); 1024 1025 /* 1026 * Kick all CPUs to make them preempt any LWPs running in user mode. 1027 * They'll trap into the kernel and suspend themselves in userret(). 1028 */ 1029 for (CPU_INFO_FOREACH(cii, ci)) { 1030 spc_lock(ci); 1031 cpu_need_resched(ci, RESCHED_IMMED); 1032 spc_unlock(ci); 1033 } 1034 } 1035 1036 /* 1037 * sched_unsleep: 1038 * 1039 * The is called when the LWP has not been awoken normally but instead 1040 * interrupted: for example, if the sleep timed out. Because of this, 1041 * it's not a valid action for running or idle LWPs. 1042 */ 1043 static void 1044 sched_unsleep(struct lwp *l, bool cleanup) 1045 { 1046 1047 lwp_unlock(l); 1048 panic("sched_unsleep"); 1049 } 1050 1051 static void 1052 resched_cpu(struct lwp *l) 1053 { 1054 struct cpu_info *ci = l->l_cpu; 1055 1056 KASSERT(lwp_locked(l, NULL)); 1057 if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority) 1058 cpu_need_resched(ci, 0); 1059 } 1060 1061 static void 1062 sched_changepri(struct lwp *l, pri_t pri) 1063 { 1064 1065 KASSERT(lwp_locked(l, NULL)); 1066 1067 if (l->l_stat == LSRUN) { 1068 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1069 sched_dequeue(l); 1070 l->l_priority = pri; 1071 sched_enqueue(l, false); 1072 } else { 1073 l->l_priority = pri; 1074 } 1075 resched_cpu(l); 1076 } 1077 1078 static void 1079 sched_lendpri(struct lwp *l, pri_t pri) 1080 { 1081 1082 KASSERT(lwp_locked(l, NULL)); 1083 1084 if (l->l_stat == LSRUN) { 1085 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1086 sched_dequeue(l); 1087 l->l_inheritedprio = pri; 1088 sched_enqueue(l, false); 1089 } else { 1090 l->l_inheritedprio = pri; 1091 } 1092 resched_cpu(l); 1093 } 1094 1095 struct lwp * 1096 syncobj_noowner(wchan_t wchan) 1097 { 1098 1099 return NULL; 1100 } 1101 1102 /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */ 1103 const fixpt_t ccpu = 0.95122942450071400909 * FSCALE; 1104 1105 /* 1106 * Constants for averages over 1, 5 and 15 minutes when sampling at 1107 * 5 second intervals. 1108 */ 1109 static const fixpt_t cexp[ ] = { 1110 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 1111 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 1112 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 1113 }; 1114 1115 /* 1116 * sched_pstats: 1117 * 1118 * => Update process statistics and check CPU resource allocation. 1119 * => Call scheduler-specific hook to eventually adjust LWP priorities. 1120 * => Compute load average of a quantity on 1, 5 and 15 minute intervals. 1121 */ 1122 void 1123 sched_pstats(void) 1124 { 1125 extern struct loadavg averunnable; 1126 struct loadavg *avg = &averunnable; 1127 const int clkhz = (stathz != 0 ? stathz : hz); 1128 static bool backwards = false; 1129 static u_int lavg_count = 0; 1130 struct proc *p; 1131 int nrun; 1132 1133 sched_pstats_ticks++; 1134 if (++lavg_count >= 5) { 1135 lavg_count = 0; 1136 nrun = 0; 1137 } 1138 mutex_enter(proc_lock); 1139 PROCLIST_FOREACH(p, &allproc) { 1140 struct lwp *l; 1141 struct rlimit *rlim; 1142 time_t runtm; 1143 int sig; 1144 1145 /* Increment sleep time (if sleeping), ignore overflow. */ 1146 mutex_enter(p->p_lock); 1147 runtm = p->p_rtime.sec; 1148 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1149 fixpt_t lpctcpu; 1150 u_int lcpticks; 1151 1152 if (__predict_false((l->l_flag & LW_IDLE) != 0)) 1153 continue; 1154 lwp_lock(l); 1155 runtm += l->l_rtime.sec; 1156 l->l_swtime++; 1157 sched_lwp_stats(l); 1158 1159 /* For load average calculation. */ 1160 if (__predict_false(lavg_count == 0) && 1161 (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) { 1162 switch (l->l_stat) { 1163 case LSSLEEP: 1164 if (l->l_slptime > 1) { 1165 break; 1166 } 1167 case LSRUN: 1168 case LSONPROC: 1169 case LSIDL: 1170 nrun++; 1171 } 1172 } 1173 lwp_unlock(l); 1174 1175 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 1176 if (l->l_slptime != 0) 1177 continue; 1178 1179 lpctcpu = l->l_pctcpu; 1180 lcpticks = atomic_swap_uint(&l->l_cpticks, 0); 1181 lpctcpu += ((FSCALE - ccpu) * 1182 (lcpticks * FSCALE / clkhz)) >> FSHIFT; 1183 l->l_pctcpu = lpctcpu; 1184 } 1185 /* Calculating p_pctcpu only for ps(1) */ 1186 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 1187 1188 if (__predict_false(runtm < 0)) { 1189 if (!backwards) { 1190 backwards = true; 1191 printf("WARNING: negative runtime; " 1192 "monotonic clock has gone backwards\n"); 1193 } 1194 mutex_exit(p->p_lock); 1195 continue; 1196 } 1197 1198 /* 1199 * Check if the process exceeds its CPU resource allocation. 1200 * If over the hard limit, kill it with SIGKILL. 1201 * If over the soft limit, send SIGXCPU and raise 1202 * the soft limit a little. 1203 */ 1204 rlim = &p->p_rlimit[RLIMIT_CPU]; 1205 sig = 0; 1206 if (__predict_false(runtm >= rlim->rlim_cur)) { 1207 if (runtm >= rlim->rlim_max) { 1208 sig = SIGKILL; 1209 log(LOG_NOTICE, "pid %d is killed: %s\n", 1210 p->p_pid, "exceeded RLIMIT_CPU"); 1211 uprintf("pid %d, command %s, is killed: %s\n", 1212 p->p_pid, p->p_comm, 1213 "exceeded RLIMIT_CPU"); 1214 } else { 1215 sig = SIGXCPU; 1216 if (rlim->rlim_cur < rlim->rlim_max) 1217 rlim->rlim_cur += 5; 1218 } 1219 } 1220 mutex_exit(p->p_lock); 1221 if (__predict_false(sig)) { 1222 KASSERT((p->p_flag & PK_SYSTEM) == 0); 1223 psignal(p, sig); 1224 } 1225 } 1226 mutex_exit(proc_lock); 1227 1228 /* Load average calculation. */ 1229 if (__predict_false(lavg_count == 0)) { 1230 int i; 1231 CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg)); 1232 for (i = 0; i < __arraycount(cexp); i++) { 1233 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 1234 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 1235 } 1236 } 1237 1238 /* Lightning bolt. */ 1239 cv_broadcast(&lbolt); 1240 } 1241