1 /* $NetBSD: kern_synch.c,v 1.298 2012/02/19 21:06:54 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 10 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 11 * Daniel Sieger. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /*- 36 * Copyright (c) 1982, 1986, 1990, 1991, 1993 37 * The Regents of the University of California. All rights reserved. 38 * (c) UNIX System Laboratories, Inc. 39 * All or some portions of this file are derived from material licensed 40 * to the University of California by American Telephone and Telegraph 41 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 42 * the permission of UNIX System Laboratories, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 3. Neither the name of the University nor the names of its contributors 53 * may be used to endorse or promote products derived from this software 54 * without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 * SUCH DAMAGE. 67 * 68 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.298 2012/02/19 21:06:54 rmind Exp $"); 73 74 #include "opt_kstack.h" 75 #include "opt_perfctrs.h" 76 #include "opt_dtrace.h" 77 78 #define __MUTEX_PRIVATE 79 80 #include <sys/param.h> 81 #include <sys/systm.h> 82 #include <sys/proc.h> 83 #include <sys/kernel.h> 84 #if defined(PERFCTRS) 85 #include <sys/pmc.h> 86 #endif 87 #include <sys/cpu.h> 88 #include <sys/pserialize.h> 89 #include <sys/resourcevar.h> 90 #include <sys/sched.h> 91 #include <sys/syscall_stats.h> 92 #include <sys/sleepq.h> 93 #include <sys/lockdebug.h> 94 #include <sys/evcnt.h> 95 #include <sys/intr.h> 96 #include <sys/lwpctl.h> 97 #include <sys/atomic.h> 98 #include <sys/simplelock.h> 99 #include <sys/syslog.h> 100 101 #include <uvm/uvm_extern.h> 102 103 #include <dev/lockstat.h> 104 105 #include <sys/dtrace_bsd.h> 106 int dtrace_vtime_active=0; 107 dtrace_vtime_switch_func_t dtrace_vtime_switch_func; 108 109 static void sched_unsleep(struct lwp *, bool); 110 static void sched_changepri(struct lwp *, pri_t); 111 static void sched_lendpri(struct lwp *, pri_t); 112 static void resched_cpu(struct lwp *); 113 114 syncobj_t sleep_syncobj = { 115 SOBJ_SLEEPQ_SORTED, 116 sleepq_unsleep, 117 sleepq_changepri, 118 sleepq_lendpri, 119 syncobj_noowner, 120 }; 121 122 syncobj_t sched_syncobj = { 123 SOBJ_SLEEPQ_SORTED, 124 sched_unsleep, 125 sched_changepri, 126 sched_lendpri, 127 syncobj_noowner, 128 }; 129 130 /* "Lightning bolt": once a second sleep address. */ 131 kcondvar_t lbolt __cacheline_aligned; 132 133 u_int sched_pstats_ticks __cacheline_aligned; 134 135 /* Preemption event counters. */ 136 static struct evcnt kpreempt_ev_crit __cacheline_aligned; 137 static struct evcnt kpreempt_ev_klock __cacheline_aligned; 138 static struct evcnt kpreempt_ev_immed __cacheline_aligned; 139 140 /* 141 * During autoconfiguration or after a panic, a sleep will simply lower the 142 * priority briefly to allow interrupts, then return. The priority to be 143 * used (safepri) is machine-dependent, thus this value is initialized and 144 * maintained in the machine-dependent layers. This priority will typically 145 * be 0, or the lowest priority that is safe for use on the interrupt stack; 146 * it can be made higher to block network software interrupts after panics. 147 */ 148 int safepri; 149 150 void 151 synch_init(void) 152 { 153 154 cv_init(&lbolt, "lbolt"); 155 156 evcnt_attach_dynamic(&kpreempt_ev_crit, EVCNT_TYPE_MISC, NULL, 157 "kpreempt", "defer: critical section"); 158 evcnt_attach_dynamic(&kpreempt_ev_klock, EVCNT_TYPE_MISC, NULL, 159 "kpreempt", "defer: kernel_lock"); 160 evcnt_attach_dynamic(&kpreempt_ev_immed, EVCNT_TYPE_MISC, NULL, 161 "kpreempt", "immediate"); 162 } 163 164 /* 165 * OBSOLETE INTERFACE 166 * 167 * General sleep call. Suspends the current LWP until a wakeup is 168 * performed on the specified identifier. The LWP will then be made 169 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 170 * means no timeout). If pri includes PCATCH flag, signals are checked 171 * before and after sleeping, else signals are not checked. Returns 0 if 172 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 173 * signal needs to be delivered, ERESTART is returned if the current system 174 * call should be restarted if possible, and EINTR is returned if the system 175 * call should be interrupted by the signal (return EINTR). 176 */ 177 int 178 tsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo) 179 { 180 struct lwp *l = curlwp; 181 sleepq_t *sq; 182 kmutex_t *mp; 183 184 KASSERT((l->l_pflag & LP_INTR) == 0); 185 KASSERT(ident != &lbolt); 186 187 if (sleepq_dontsleep(l)) { 188 (void)sleepq_abort(NULL, 0); 189 return 0; 190 } 191 192 l->l_kpriority = true; 193 sq = sleeptab_lookup(&sleeptab, ident, &mp); 194 sleepq_enter(sq, l, mp); 195 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 196 return sleepq_block(timo, priority & PCATCH); 197 } 198 199 int 200 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 201 kmutex_t *mtx) 202 { 203 struct lwp *l = curlwp; 204 sleepq_t *sq; 205 kmutex_t *mp; 206 int error; 207 208 KASSERT((l->l_pflag & LP_INTR) == 0); 209 KASSERT(ident != &lbolt); 210 211 if (sleepq_dontsleep(l)) { 212 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 213 return 0; 214 } 215 216 l->l_kpriority = true; 217 sq = sleeptab_lookup(&sleeptab, ident, &mp); 218 sleepq_enter(sq, l, mp); 219 sleepq_enqueue(sq, ident, wmesg, &sleep_syncobj); 220 mutex_exit(mtx); 221 error = sleepq_block(timo, priority & PCATCH); 222 223 if ((priority & PNORELOCK) == 0) 224 mutex_enter(mtx); 225 226 return error; 227 } 228 229 /* 230 * General sleep call for situations where a wake-up is not expected. 231 */ 232 int 233 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 234 { 235 struct lwp *l = curlwp; 236 kmutex_t *mp; 237 sleepq_t *sq; 238 int error; 239 240 KASSERT(!(timo == 0 && intr == false)); 241 242 if (sleepq_dontsleep(l)) 243 return sleepq_abort(NULL, 0); 244 245 if (mtx != NULL) 246 mutex_exit(mtx); 247 l->l_kpriority = true; 248 sq = sleeptab_lookup(&sleeptab, l, &mp); 249 sleepq_enter(sq, l, mp); 250 sleepq_enqueue(sq, l, wmesg, &sleep_syncobj); 251 error = sleepq_block(timo, intr); 252 if (mtx != NULL) 253 mutex_enter(mtx); 254 255 return error; 256 } 257 258 /* 259 * OBSOLETE INTERFACE 260 * 261 * Make all LWPs sleeping on the specified identifier runnable. 262 */ 263 void 264 wakeup(wchan_t ident) 265 { 266 sleepq_t *sq; 267 kmutex_t *mp; 268 269 if (__predict_false(cold)) 270 return; 271 272 sq = sleeptab_lookup(&sleeptab, ident, &mp); 273 sleepq_wake(sq, ident, (u_int)-1, mp); 274 } 275 276 /* 277 * General yield call. Puts the current LWP back on its run queue and 278 * performs a voluntary context switch. Should only be called when the 279 * current LWP explicitly requests it (eg sched_yield(2)). 280 */ 281 void 282 yield(void) 283 { 284 struct lwp *l = curlwp; 285 286 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 287 lwp_lock(l); 288 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 289 KASSERT(l->l_stat == LSONPROC); 290 l->l_kpriority = false; 291 (void)mi_switch(l); 292 KERNEL_LOCK(l->l_biglocks, l); 293 } 294 295 /* 296 * General preemption call. Puts the current LWP back on its run queue 297 * and performs an involuntary context switch. 298 */ 299 void 300 preempt(void) 301 { 302 struct lwp *l = curlwp; 303 304 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 305 lwp_lock(l); 306 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_lwplock)); 307 KASSERT(l->l_stat == LSONPROC); 308 l->l_kpriority = false; 309 l->l_nivcsw++; 310 (void)mi_switch(l); 311 KERNEL_LOCK(l->l_biglocks, l); 312 } 313 314 /* 315 * Handle a request made by another agent to preempt the current LWP 316 * in-kernel. Usually called when l_dopreempt may be non-zero. 317 * 318 * Character addresses for lockstat only. 319 */ 320 static char in_critical_section; 321 static char kernel_lock_held; 322 static char is_softint; 323 static char cpu_kpreempt_enter_fail; 324 325 bool 326 kpreempt(uintptr_t where) 327 { 328 uintptr_t failed; 329 lwp_t *l; 330 int s, dop, lsflag; 331 332 l = curlwp; 333 failed = 0; 334 while ((dop = l->l_dopreempt) != 0) { 335 if (l->l_stat != LSONPROC) { 336 /* 337 * About to block (or die), let it happen. 338 * Doesn't really count as "preemption has 339 * been blocked", since we're going to 340 * context switch. 341 */ 342 l->l_dopreempt = 0; 343 return true; 344 } 345 if (__predict_false((l->l_flag & LW_IDLE) != 0)) { 346 /* Can't preempt idle loop, don't count as failure. */ 347 l->l_dopreempt = 0; 348 return true; 349 } 350 if (__predict_false(l->l_nopreempt != 0)) { 351 /* LWP holds preemption disabled, explicitly. */ 352 if ((dop & DOPREEMPT_COUNTED) == 0) { 353 kpreempt_ev_crit.ev_count++; 354 } 355 failed = (uintptr_t)&in_critical_section; 356 break; 357 } 358 if (__predict_false((l->l_pflag & LP_INTR) != 0)) { 359 /* Can't preempt soft interrupts yet. */ 360 l->l_dopreempt = 0; 361 failed = (uintptr_t)&is_softint; 362 break; 363 } 364 s = splsched(); 365 if (__predict_false(l->l_blcnt != 0 || 366 curcpu()->ci_biglock_wanted != NULL)) { 367 /* Hold or want kernel_lock, code is not MT safe. */ 368 splx(s); 369 if ((dop & DOPREEMPT_COUNTED) == 0) { 370 kpreempt_ev_klock.ev_count++; 371 } 372 failed = (uintptr_t)&kernel_lock_held; 373 break; 374 } 375 if (__predict_false(!cpu_kpreempt_enter(where, s))) { 376 /* 377 * It may be that the IPL is too high. 378 * kpreempt_enter() can schedule an 379 * interrupt to retry later. 380 */ 381 splx(s); 382 failed = (uintptr_t)&cpu_kpreempt_enter_fail; 383 break; 384 } 385 /* Do it! */ 386 if (__predict_true((dop & DOPREEMPT_COUNTED) == 0)) { 387 kpreempt_ev_immed.ev_count++; 388 } 389 lwp_lock(l); 390 mi_switch(l); 391 l->l_nopreempt++; 392 splx(s); 393 394 /* Take care of any MD cleanup. */ 395 cpu_kpreempt_exit(where); 396 l->l_nopreempt--; 397 } 398 399 if (__predict_true(!failed)) { 400 return false; 401 } 402 403 /* Record preemption failure for reporting via lockstat. */ 404 atomic_or_uint(&l->l_dopreempt, DOPREEMPT_COUNTED); 405 lsflag = 0; 406 LOCKSTAT_ENTER(lsflag); 407 if (__predict_false(lsflag)) { 408 if (where == 0) { 409 where = (uintptr_t)__builtin_return_address(0); 410 } 411 /* Preemption is on, might recurse, so make it atomic. */ 412 if (atomic_cas_ptr_ni((void *)&l->l_pfailaddr, NULL, 413 (void *)where) == NULL) { 414 LOCKSTAT_START_TIMER(lsflag, l->l_pfailtime); 415 l->l_pfaillock = failed; 416 } 417 } 418 LOCKSTAT_EXIT(lsflag); 419 return true; 420 } 421 422 /* 423 * Return true if preemption is explicitly disabled. 424 */ 425 bool 426 kpreempt_disabled(void) 427 { 428 const lwp_t *l = curlwp; 429 430 return l->l_nopreempt != 0 || l->l_stat == LSZOMB || 431 (l->l_flag & LW_IDLE) != 0 || cpu_kpreempt_disabled(); 432 } 433 434 /* 435 * Disable kernel preemption. 436 */ 437 void 438 kpreempt_disable(void) 439 { 440 441 KPREEMPT_DISABLE(curlwp); 442 } 443 444 /* 445 * Reenable kernel preemption. 446 */ 447 void 448 kpreempt_enable(void) 449 { 450 451 KPREEMPT_ENABLE(curlwp); 452 } 453 454 /* 455 * Compute the amount of time during which the current lwp was running. 456 * 457 * - update l_rtime unless it's an idle lwp. 458 */ 459 460 void 461 updatertime(lwp_t *l, const struct bintime *now) 462 { 463 464 if (__predict_false(l->l_flag & LW_IDLE)) 465 return; 466 467 /* rtime += now - stime */ 468 bintime_add(&l->l_rtime, now); 469 bintime_sub(&l->l_rtime, &l->l_stime); 470 } 471 472 /* 473 * Select next LWP from the current CPU to run.. 474 */ 475 static inline lwp_t * 476 nextlwp(struct cpu_info *ci, struct schedstate_percpu *spc) 477 { 478 lwp_t *newl; 479 480 /* 481 * Let sched_nextlwp() select the LWP to run the CPU next. 482 * If no LWP is runnable, select the idle LWP. 483 * 484 * Note that spc_lwplock might not necessary be held, and 485 * new thread would be unlocked after setting the LWP-lock. 486 */ 487 newl = sched_nextlwp(); 488 if (newl != NULL) { 489 sched_dequeue(newl); 490 KASSERT(lwp_locked(newl, spc->spc_mutex)); 491 KASSERT(newl->l_cpu == ci); 492 newl->l_stat = LSONPROC; 493 newl->l_pflag |= LP_RUNNING; 494 lwp_setlock(newl, spc->spc_lwplock); 495 } else { 496 newl = ci->ci_data.cpu_idlelwp; 497 newl->l_stat = LSONPROC; 498 newl->l_pflag |= LP_RUNNING; 499 } 500 501 /* 502 * Only clear want_resched if there are no pending (slow) 503 * software interrupts. 504 */ 505 ci->ci_want_resched = ci->ci_data.cpu_softints; 506 spc->spc_flags &= ~SPCF_SWITCHCLEAR; 507 spc->spc_curpriority = lwp_eprio(newl); 508 509 return newl; 510 } 511 512 /* 513 * The machine independent parts of context switch. 514 * 515 * Returns 1 if another LWP was actually run. 516 */ 517 int 518 mi_switch(lwp_t *l) 519 { 520 struct cpu_info *ci; 521 struct schedstate_percpu *spc; 522 struct lwp *newl; 523 int retval, oldspl; 524 struct bintime bt; 525 bool returning; 526 527 KASSERT(lwp_locked(l, NULL)); 528 KASSERT(kpreempt_disabled()); 529 LOCKDEBUG_BARRIER(l->l_mutex, 1); 530 531 kstack_check_magic(l); 532 533 binuptime(&bt); 534 535 KASSERT((l->l_pflag & LP_RUNNING) != 0); 536 KASSERT(l->l_cpu == curcpu()); 537 ci = l->l_cpu; 538 spc = &ci->ci_schedstate; 539 returning = false; 540 newl = NULL; 541 542 /* 543 * If we have been asked to switch to a specific LWP, then there 544 * is no need to inspect the run queues. If a soft interrupt is 545 * blocking, then return to the interrupted thread without adjusting 546 * VM context or its start time: neither have been changed in order 547 * to take the interrupt. 548 */ 549 if (l->l_switchto != NULL) { 550 if ((l->l_pflag & LP_INTR) != 0) { 551 returning = true; 552 softint_block(l); 553 if ((l->l_pflag & LP_TIMEINTR) != 0) 554 updatertime(l, &bt); 555 } 556 newl = l->l_switchto; 557 l->l_switchto = NULL; 558 } 559 #ifndef __HAVE_FAST_SOFTINTS 560 else if (ci->ci_data.cpu_softints != 0) { 561 /* There are pending soft interrupts, so pick one. */ 562 newl = softint_picklwp(); 563 newl->l_stat = LSONPROC; 564 newl->l_pflag |= LP_RUNNING; 565 } 566 #endif /* !__HAVE_FAST_SOFTINTS */ 567 568 /* Count time spent in current system call */ 569 if (!returning) { 570 SYSCALL_TIME_SLEEP(l); 571 572 /* 573 * XXXSMP If we are using h/w performance counters, 574 * save context. 575 */ 576 #if PERFCTRS 577 if (PMC_ENABLED(l->l_proc)) { 578 pmc_save_context(l->l_proc); 579 } 580 #endif 581 updatertime(l, &bt); 582 } 583 584 /* Lock the runqueue */ 585 KASSERT(l->l_stat != LSRUN); 586 mutex_spin_enter(spc->spc_mutex); 587 588 /* 589 * If on the CPU and we have gotten this far, then we must yield. 590 */ 591 if (l->l_stat == LSONPROC && l != newl) { 592 KASSERT(lwp_locked(l, spc->spc_lwplock)); 593 if ((l->l_flag & LW_IDLE) == 0) { 594 l->l_stat = LSRUN; 595 lwp_setlock(l, spc->spc_mutex); 596 sched_enqueue(l, true); 597 /* 598 * Handle migration. Note that "migrating LWP" may 599 * be reset here, if interrupt/preemption happens 600 * early in idle LWP. 601 */ 602 if (l->l_target_cpu != NULL) { 603 KASSERT((l->l_pflag & LP_INTR) == 0); 604 spc->spc_migrating = l; 605 } 606 } else 607 l->l_stat = LSIDL; 608 } 609 610 /* Pick new LWP to run. */ 611 if (newl == NULL) { 612 newl = nextlwp(ci, spc); 613 } 614 615 /* Items that must be updated with the CPU locked. */ 616 if (!returning) { 617 /* Update the new LWP's start time. */ 618 newl->l_stime = bt; 619 620 /* 621 * ci_curlwp changes when a fast soft interrupt occurs. 622 * We use cpu_onproc to keep track of which kernel or 623 * user thread is running 'underneath' the software 624 * interrupt. This is important for time accounting, 625 * itimers and forcing user threads to preempt (aston). 626 */ 627 ci->ci_data.cpu_onproc = newl; 628 } 629 630 /* 631 * Preemption related tasks. Must be done with the current 632 * CPU locked. 633 */ 634 cpu_did_resched(l); 635 l->l_dopreempt = 0; 636 if (__predict_false(l->l_pfailaddr != 0)) { 637 LOCKSTAT_FLAG(lsflag); 638 LOCKSTAT_ENTER(lsflag); 639 LOCKSTAT_STOP_TIMER(lsflag, l->l_pfailtime); 640 LOCKSTAT_EVENT_RA(lsflag, l->l_pfaillock, LB_NOPREEMPT|LB_SPIN, 641 1, l->l_pfailtime, l->l_pfailaddr); 642 LOCKSTAT_EXIT(lsflag); 643 l->l_pfailtime = 0; 644 l->l_pfaillock = 0; 645 l->l_pfailaddr = 0; 646 } 647 648 if (l != newl) { 649 struct lwp *prevlwp; 650 651 /* Release all locks, but leave the current LWP locked */ 652 if (l->l_mutex == spc->spc_mutex) { 653 /* 654 * Drop spc_lwplock, if the current LWP has been moved 655 * to the run queue (it is now locked by spc_mutex). 656 */ 657 mutex_spin_exit(spc->spc_lwplock); 658 } else { 659 /* 660 * Otherwise, drop the spc_mutex, we are done with the 661 * run queues. 662 */ 663 mutex_spin_exit(spc->spc_mutex); 664 } 665 666 /* 667 * Mark that context switch is going to be performed 668 * for this LWP, to protect it from being switched 669 * to on another CPU. 670 */ 671 KASSERT(l->l_ctxswtch == 0); 672 l->l_ctxswtch = 1; 673 l->l_ncsw++; 674 KASSERT((l->l_pflag & LP_RUNNING) != 0); 675 l->l_pflag &= ~LP_RUNNING; 676 677 /* 678 * Increase the count of spin-mutexes before the release 679 * of the last lock - we must remain at IPL_SCHED during 680 * the context switch. 681 */ 682 KASSERTMSG(ci->ci_mtx_count == -1, 683 "%s: cpu%u: ci_mtx_count (%d) != -1", 684 __func__, cpu_index(ci), ci->ci_mtx_count); 685 oldspl = MUTEX_SPIN_OLDSPL(ci); 686 ci->ci_mtx_count--; 687 lwp_unlock(l); 688 689 /* Count the context switch on this CPU. */ 690 ci->ci_data.cpu_nswtch++; 691 692 /* Update status for lwpctl, if present. */ 693 if (l->l_lwpctl != NULL) 694 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_NONE; 695 696 /* 697 * Save old VM context, unless a soft interrupt 698 * handler is blocking. 699 */ 700 if (!returning) 701 pmap_deactivate(l); 702 703 /* 704 * We may need to spin-wait if 'newl' is still 705 * context switching on another CPU. 706 */ 707 if (__predict_false(newl->l_ctxswtch != 0)) { 708 u_int count; 709 count = SPINLOCK_BACKOFF_MIN; 710 while (newl->l_ctxswtch) 711 SPINLOCK_BACKOFF(count); 712 } 713 714 /* 715 * If DTrace has set the active vtime enum to anything 716 * other than INACTIVE (0), then it should have set the 717 * function to call. 718 */ 719 if (__predict_false(dtrace_vtime_active)) { 720 (*dtrace_vtime_switch_func)(newl); 721 } 722 723 /* Switch to the new LWP.. */ 724 prevlwp = cpu_switchto(l, newl, returning); 725 ci = curcpu(); 726 727 /* 728 * Switched away - we have new curlwp. 729 * Restore VM context and IPL. 730 */ 731 pmap_activate(l); 732 uvm_emap_switch(l); 733 pcu_switchpoint(l); 734 735 if (prevlwp != NULL) { 736 /* Normalize the count of the spin-mutexes */ 737 ci->ci_mtx_count++; 738 /* Unmark the state of context switch */ 739 membar_exit(); 740 prevlwp->l_ctxswtch = 0; 741 } 742 743 /* Update status for lwpctl, if present. */ 744 if (l->l_lwpctl != NULL) { 745 l->l_lwpctl->lc_curcpu = (int)cpu_index(ci); 746 l->l_lwpctl->lc_pctr++; 747 } 748 749 /* Note trip through cpu_switchto(). */ 750 pserialize_switchpoint(); 751 752 KASSERT(l->l_cpu == ci); 753 splx(oldspl); 754 retval = 1; 755 } else { 756 /* Nothing to do - just unlock and return. */ 757 mutex_spin_exit(spc->spc_mutex); 758 lwp_unlock(l); 759 retval = 0; 760 } 761 762 KASSERT(l == curlwp); 763 KASSERT(l->l_stat == LSONPROC); 764 765 /* 766 * XXXSMP If we are using h/w performance counters, restore context. 767 * XXXSMP preemption problem. 768 */ 769 #if PERFCTRS 770 if (PMC_ENABLED(l->l_proc)) { 771 pmc_restore_context(l->l_proc); 772 } 773 #endif 774 SYSCALL_TIME_WAKEUP(l); 775 LOCKDEBUG_BARRIER(NULL, 1); 776 777 return retval; 778 } 779 780 /* 781 * The machine independent parts of context switch to oblivion. 782 * Does not return. Call with the LWP unlocked. 783 */ 784 void 785 lwp_exit_switchaway(lwp_t *l) 786 { 787 struct cpu_info *ci; 788 struct lwp *newl; 789 struct bintime bt; 790 791 ci = l->l_cpu; 792 793 KASSERT(kpreempt_disabled()); 794 KASSERT(l->l_stat == LSZOMB || l->l_stat == LSIDL); 795 KASSERT(ci == curcpu()); 796 LOCKDEBUG_BARRIER(NULL, 0); 797 798 kstack_check_magic(l); 799 800 /* Count time spent in current system call */ 801 SYSCALL_TIME_SLEEP(l); 802 binuptime(&bt); 803 updatertime(l, &bt); 804 805 /* Must stay at IPL_SCHED even after releasing run queue lock. */ 806 (void)splsched(); 807 808 /* 809 * Let sched_nextlwp() select the LWP to run the CPU next. 810 * If no LWP is runnable, select the idle LWP. 811 * 812 * Note that spc_lwplock might not necessary be held, and 813 * new thread would be unlocked after setting the LWP-lock. 814 */ 815 spc_lock(ci); 816 #ifndef __HAVE_FAST_SOFTINTS 817 if (ci->ci_data.cpu_softints != 0) { 818 /* There are pending soft interrupts, so pick one. */ 819 newl = softint_picklwp(); 820 newl->l_stat = LSONPROC; 821 newl->l_pflag |= LP_RUNNING; 822 } else 823 #endif /* !__HAVE_FAST_SOFTINTS */ 824 { 825 newl = nextlwp(ci, &ci->ci_schedstate); 826 } 827 828 /* Update the new LWP's start time. */ 829 newl->l_stime = bt; 830 l->l_pflag &= ~LP_RUNNING; 831 832 /* 833 * ci_curlwp changes when a fast soft interrupt occurs. 834 * We use cpu_onproc to keep track of which kernel or 835 * user thread is running 'underneath' the software 836 * interrupt. This is important for time accounting, 837 * itimers and forcing user threads to preempt (aston). 838 */ 839 ci->ci_data.cpu_onproc = newl; 840 841 /* 842 * Preemption related tasks. Must be done with the current 843 * CPU locked. 844 */ 845 cpu_did_resched(l); 846 847 /* Unlock the run queue. */ 848 spc_unlock(ci); 849 850 /* Count the context switch on this CPU. */ 851 ci->ci_data.cpu_nswtch++; 852 853 /* Update status for lwpctl, if present. */ 854 if (l->l_lwpctl != NULL) 855 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 856 857 /* 858 * We may need to spin-wait if 'newl' is still 859 * context switching on another CPU. 860 */ 861 if (__predict_false(newl->l_ctxswtch != 0)) { 862 u_int count; 863 count = SPINLOCK_BACKOFF_MIN; 864 while (newl->l_ctxswtch) 865 SPINLOCK_BACKOFF(count); 866 } 867 868 /* 869 * If DTrace has set the active vtime enum to anything 870 * other than INACTIVE (0), then it should have set the 871 * function to call. 872 */ 873 if (__predict_false(dtrace_vtime_active)) { 874 (*dtrace_vtime_switch_func)(newl); 875 } 876 877 /* Switch to the new LWP.. */ 878 (void)cpu_switchto(NULL, newl, false); 879 880 for (;;) continue; /* XXX: convince gcc about "noreturn" */ 881 /* NOTREACHED */ 882 } 883 884 /* 885 * setrunnable: change LWP state to be runnable, placing it on the run queue. 886 * 887 * Call with the process and LWP locked. Will return with the LWP unlocked. 888 */ 889 void 890 setrunnable(struct lwp *l) 891 { 892 struct proc *p = l->l_proc; 893 struct cpu_info *ci; 894 895 KASSERT((l->l_flag & LW_IDLE) == 0); 896 KASSERT(mutex_owned(p->p_lock)); 897 KASSERT(lwp_locked(l, NULL)); 898 KASSERT(l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex); 899 900 switch (l->l_stat) { 901 case LSSTOP: 902 /* 903 * If we're being traced (possibly because someone attached us 904 * while we were stopped), check for a signal from the debugger. 905 */ 906 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0) 907 signotify(l); 908 p->p_nrlwps++; 909 break; 910 case LSSUSPENDED: 911 l->l_flag &= ~LW_WSUSPEND; 912 p->p_nrlwps++; 913 cv_broadcast(&p->p_lwpcv); 914 break; 915 case LSSLEEP: 916 KASSERT(l->l_wchan != NULL); 917 break; 918 default: 919 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 920 } 921 922 /* 923 * If the LWP was sleeping, start it again. 924 */ 925 if (l->l_wchan != NULL) { 926 l->l_stat = LSSLEEP; 927 /* lwp_unsleep() will release the lock. */ 928 lwp_unsleep(l, true); 929 return; 930 } 931 932 /* 933 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 934 * about to call mi_switch(), in which case it will yield. 935 */ 936 if ((l->l_pflag & LP_RUNNING) != 0) { 937 l->l_stat = LSONPROC; 938 l->l_slptime = 0; 939 lwp_unlock(l); 940 return; 941 } 942 943 /* 944 * Look for a CPU to run. 945 * Set the LWP runnable. 946 */ 947 ci = sched_takecpu(l); 948 l->l_cpu = ci; 949 spc_lock(ci); 950 lwp_unlock_to(l, ci->ci_schedstate.spc_mutex); 951 sched_setrunnable(l); 952 l->l_stat = LSRUN; 953 l->l_slptime = 0; 954 955 sched_enqueue(l, false); 956 resched_cpu(l); 957 lwp_unlock(l); 958 } 959 960 /* 961 * suspendsched: 962 * 963 * Convert all non-LW_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 964 */ 965 void 966 suspendsched(void) 967 { 968 CPU_INFO_ITERATOR cii; 969 struct cpu_info *ci; 970 struct lwp *l; 971 struct proc *p; 972 973 /* 974 * We do this by process in order not to violate the locking rules. 975 */ 976 mutex_enter(proc_lock); 977 PROCLIST_FOREACH(p, &allproc) { 978 mutex_enter(p->p_lock); 979 if ((p->p_flag & PK_SYSTEM) != 0) { 980 mutex_exit(p->p_lock); 981 continue; 982 } 983 984 p->p_stat = SSTOP; 985 986 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 987 if (l == curlwp) 988 continue; 989 990 lwp_lock(l); 991 992 /* 993 * Set L_WREBOOT so that the LWP will suspend itself 994 * when it tries to return to user mode. We want to 995 * try and get to get as many LWPs as possible to 996 * the user / kernel boundary, so that they will 997 * release any locks that they hold. 998 */ 999 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 1000 1001 if (l->l_stat == LSSLEEP && 1002 (l->l_flag & LW_SINTR) != 0) { 1003 /* setrunnable() will release the lock. */ 1004 setrunnable(l); 1005 continue; 1006 } 1007 1008 lwp_unlock(l); 1009 } 1010 1011 mutex_exit(p->p_lock); 1012 } 1013 mutex_exit(proc_lock); 1014 1015 /* 1016 * Kick all CPUs to make them preempt any LWPs running in user mode. 1017 * They'll trap into the kernel and suspend themselves in userret(). 1018 */ 1019 for (CPU_INFO_FOREACH(cii, ci)) { 1020 spc_lock(ci); 1021 cpu_need_resched(ci, RESCHED_IMMED); 1022 spc_unlock(ci); 1023 } 1024 } 1025 1026 /* 1027 * sched_unsleep: 1028 * 1029 * The is called when the LWP has not been awoken normally but instead 1030 * interrupted: for example, if the sleep timed out. Because of this, 1031 * it's not a valid action for running or idle LWPs. 1032 */ 1033 static void 1034 sched_unsleep(struct lwp *l, bool cleanup) 1035 { 1036 1037 lwp_unlock(l); 1038 panic("sched_unsleep"); 1039 } 1040 1041 static void 1042 resched_cpu(struct lwp *l) 1043 { 1044 struct cpu_info *ci = l->l_cpu; 1045 1046 KASSERT(lwp_locked(l, NULL)); 1047 if (lwp_eprio(l) > ci->ci_schedstate.spc_curpriority) 1048 cpu_need_resched(ci, 0); 1049 } 1050 1051 static void 1052 sched_changepri(struct lwp *l, pri_t pri) 1053 { 1054 1055 KASSERT(lwp_locked(l, NULL)); 1056 1057 if (l->l_stat == LSRUN) { 1058 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1059 sched_dequeue(l); 1060 l->l_priority = pri; 1061 sched_enqueue(l, false); 1062 } else { 1063 l->l_priority = pri; 1064 } 1065 resched_cpu(l); 1066 } 1067 1068 static void 1069 sched_lendpri(struct lwp *l, pri_t pri) 1070 { 1071 1072 KASSERT(lwp_locked(l, NULL)); 1073 1074 if (l->l_stat == LSRUN) { 1075 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 1076 sched_dequeue(l); 1077 l->l_inheritedprio = pri; 1078 sched_enqueue(l, false); 1079 } else { 1080 l->l_inheritedprio = pri; 1081 } 1082 resched_cpu(l); 1083 } 1084 1085 struct lwp * 1086 syncobj_noowner(wchan_t wchan) 1087 { 1088 1089 return NULL; 1090 } 1091 1092 /* Decay 95% of proc::p_pctcpu in 60 seconds, ccpu = exp(-1/20) */ 1093 const fixpt_t ccpu = 0.95122942450071400909 * FSCALE; 1094 1095 /* 1096 * Constants for averages over 1, 5 and 15 minutes when sampling at 1097 * 5 second intervals. 1098 */ 1099 static const fixpt_t cexp[ ] = { 1100 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 1101 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 1102 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 1103 }; 1104 1105 /* 1106 * sched_pstats: 1107 * 1108 * => Update process statistics and check CPU resource allocation. 1109 * => Call scheduler-specific hook to eventually adjust LWP priorities. 1110 * => Compute load average of a quantity on 1, 5 and 15 minute intervals. 1111 */ 1112 void 1113 sched_pstats(void) 1114 { 1115 extern struct loadavg averunnable; 1116 struct loadavg *avg = &averunnable; 1117 const int clkhz = (stathz != 0 ? stathz : hz); 1118 static bool backwards = false; 1119 static u_int lavg_count = 0; 1120 struct proc *p; 1121 int nrun; 1122 1123 sched_pstats_ticks++; 1124 if (++lavg_count >= 5) { 1125 lavg_count = 0; 1126 nrun = 0; 1127 } 1128 mutex_enter(proc_lock); 1129 PROCLIST_FOREACH(p, &allproc) { 1130 struct lwp *l; 1131 struct rlimit *rlim; 1132 time_t runtm; 1133 int sig; 1134 1135 /* Increment sleep time (if sleeping), ignore overflow. */ 1136 mutex_enter(p->p_lock); 1137 runtm = p->p_rtime.sec; 1138 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1139 fixpt_t lpctcpu; 1140 u_int lcpticks; 1141 1142 if (__predict_false((l->l_flag & LW_IDLE) != 0)) 1143 continue; 1144 lwp_lock(l); 1145 runtm += l->l_rtime.sec; 1146 l->l_swtime++; 1147 sched_lwp_stats(l); 1148 1149 /* For load average calculation. */ 1150 if (__predict_false(lavg_count == 0) && 1151 (l->l_flag & (LW_SINTR | LW_SYSTEM)) == 0) { 1152 switch (l->l_stat) { 1153 case LSSLEEP: 1154 if (l->l_slptime > 1) { 1155 break; 1156 } 1157 case LSRUN: 1158 case LSONPROC: 1159 case LSIDL: 1160 nrun++; 1161 } 1162 } 1163 lwp_unlock(l); 1164 1165 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 1166 if (l->l_slptime != 0) 1167 continue; 1168 1169 lpctcpu = l->l_pctcpu; 1170 lcpticks = atomic_swap_uint(&l->l_cpticks, 0); 1171 lpctcpu += ((FSCALE - ccpu) * 1172 (lcpticks * FSCALE / clkhz)) >> FSHIFT; 1173 l->l_pctcpu = lpctcpu; 1174 } 1175 /* Calculating p_pctcpu only for ps(1) */ 1176 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 1177 1178 /* 1179 * Check if the process exceeds its CPU resource allocation. 1180 * If over the hard limit, kill it with SIGKILL. 1181 * If over the soft limit, send SIGXCPU and raise 1182 * the soft limit a little. 1183 */ 1184 rlim = &p->p_rlimit[RLIMIT_CPU]; 1185 sig = 0; 1186 if (__predict_false(runtm >= rlim->rlim_cur)) { 1187 if (runtm >= rlim->rlim_max) { 1188 sig = SIGKILL; 1189 log(LOG_NOTICE, "pid %d is killed: %s\n", 1190 p->p_pid, "exceeded RLIMIT_CPU"); 1191 uprintf("pid %d, command %s, is killed: %s\n", 1192 p->p_pid, p->p_comm, 1193 "exceeded RLIMIT_CPU"); 1194 } else { 1195 sig = SIGXCPU; 1196 if (rlim->rlim_cur < rlim->rlim_max) 1197 rlim->rlim_cur += 5; 1198 } 1199 } 1200 mutex_exit(p->p_lock); 1201 if (__predict_false(runtm < 0)) { 1202 if (!backwards) { 1203 backwards = true; 1204 printf("WARNING: negative runtime; " 1205 "monotonic clock has gone backwards\n"); 1206 } 1207 } else if (__predict_false(sig)) { 1208 KASSERT((p->p_flag & PK_SYSTEM) == 0); 1209 psignal(p, sig); 1210 } 1211 } 1212 mutex_exit(proc_lock); 1213 1214 /* Load average calculation. */ 1215 if (__predict_false(lavg_count == 0)) { 1216 int i; 1217 CTASSERT(__arraycount(cexp) == __arraycount(avg->ldavg)); 1218 for (i = 0; i < __arraycount(cexp); i++) { 1219 avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 1220 nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 1221 } 1222 } 1223 1224 /* Lightning bolt. */ 1225 cv_broadcast(&lbolt); 1226 } 1227