1 /* $NetBSD: kern_synch.c,v 1.194 2007/08/06 11:48:23 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2004, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, by Charles M. Hannum, Andrew Doran and 10 * Daniel Sieger. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the NetBSD 23 * Foundation, Inc. and its contributors. 24 * 4. Neither the name of The NetBSD Foundation nor the names of its 25 * contributors may be used to endorse or promote products derived 26 * from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 29 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 30 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 31 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 32 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 38 * POSSIBILITY OF SUCH DAMAGE. 39 */ 40 41 /*- 42 * Copyright (c) 1982, 1986, 1990, 1991, 1993 43 * The Regents of the University of California. All rights reserved. 44 * (c) UNIX System Laboratories, Inc. 45 * All or some portions of this file are derived from material licensed 46 * to the University of California by American Telephone and Telegraph 47 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 48 * the permission of UNIX System Laboratories, Inc. 49 * 50 * Redistribution and use in source and binary forms, with or without 51 * modification, are permitted provided that the following conditions 52 * are met: 53 * 1. Redistributions of source code must retain the above copyright 54 * notice, this list of conditions and the following disclaimer. 55 * 2. Redistributions in binary form must reproduce the above copyright 56 * notice, this list of conditions and the following disclaimer in the 57 * documentation and/or other materials provided with the distribution. 58 * 3. Neither the name of the University nor the names of its contributors 59 * may be used to endorse or promote products derived from this software 60 * without specific prior written permission. 61 * 62 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 63 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 64 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 65 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 66 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 67 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 68 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 69 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 70 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 71 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 72 * SUCH DAMAGE. 73 * 74 * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 75 */ 76 77 #include <sys/cdefs.h> 78 __KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.194 2007/08/06 11:48:23 yamt Exp $"); 79 80 #include "opt_kstack.h" 81 #include "opt_lockdebug.h" 82 #include "opt_multiprocessor.h" 83 #include "opt_perfctrs.h" 84 85 #define __MUTEX_PRIVATE 86 87 #include <sys/param.h> 88 #include <sys/systm.h> 89 #include <sys/proc.h> 90 #include <sys/kernel.h> 91 #if defined(PERFCTRS) 92 #include <sys/pmc.h> 93 #endif 94 #include <sys/cpu.h> 95 #include <sys/resourcevar.h> 96 #include <sys/sched.h> 97 #include <sys/syscall_stats.h> 98 #include <sys/sleepq.h> 99 #include <sys/lockdebug.h> 100 #include <sys/evcnt.h> 101 102 #include <uvm/uvm_extern.h> 103 104 callout_t sched_pstats_ch; 105 unsigned int sched_pstats_ticks; 106 107 kcondvar_t lbolt; /* once a second sleep address */ 108 109 static void sched_unsleep(struct lwp *); 110 static void sched_changepri(struct lwp *, pri_t); 111 static void sched_lendpri(struct lwp *, pri_t); 112 113 syncobj_t sleep_syncobj = { 114 SOBJ_SLEEPQ_SORTED, 115 sleepq_unsleep, 116 sleepq_changepri, 117 sleepq_lendpri, 118 syncobj_noowner, 119 }; 120 121 syncobj_t sched_syncobj = { 122 SOBJ_SLEEPQ_SORTED, 123 sched_unsleep, 124 sched_changepri, 125 sched_lendpri, 126 syncobj_noowner, 127 }; 128 129 /* 130 * During autoconfiguration or after a panic, a sleep will simply lower the 131 * priority briefly to allow interrupts, then return. The priority to be 132 * used (safepri) is machine-dependent, thus this value is initialized and 133 * maintained in the machine-dependent layers. This priority will typically 134 * be 0, or the lowest priority that is safe for use on the interrupt stack; 135 * it can be made higher to block network software interrupts after panics. 136 */ 137 int safepri; 138 139 /* 140 * OBSOLETE INTERFACE 141 * 142 * General sleep call. Suspends the current process until a wakeup is 143 * performed on the specified identifier. The process will then be made 144 * runnable with the specified priority. Sleeps at most timo/hz seconds (0 145 * means no timeout). If pri includes PCATCH flag, signals are checked 146 * before and after sleeping, else signals are not checked. Returns 0 if 147 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 148 * signal needs to be delivered, ERESTART is returned if the current system 149 * call should be restarted if possible, and EINTR is returned if the system 150 * call should be interrupted by the signal (return EINTR). 151 * 152 * The interlock is held until we are on a sleep queue. The interlock will 153 * be locked before returning back to the caller unless the PNORELOCK flag 154 * is specified, in which case the interlock will always be unlocked upon 155 * return. 156 */ 157 int 158 ltsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 159 volatile struct simplelock *interlock) 160 { 161 struct lwp *l = curlwp; 162 sleepq_t *sq; 163 int error; 164 165 if (sleepq_dontsleep(l)) { 166 (void)sleepq_abort(NULL, 0); 167 if ((priority & PNORELOCK) != 0) 168 simple_unlock(interlock); 169 return 0; 170 } 171 172 sq = sleeptab_lookup(&sleeptab, ident); 173 sleepq_enter(sq, l); 174 sleepq_enqueue(sq, priority & PRIMASK, ident, wmesg, &sleep_syncobj); 175 176 if (interlock != NULL) { 177 LOCK_ASSERT(simple_lock_held(interlock)); 178 simple_unlock(interlock); 179 } 180 181 error = sleepq_block(timo, priority & PCATCH); 182 183 if (interlock != NULL && (priority & PNORELOCK) == 0) 184 simple_lock(interlock); 185 186 return error; 187 } 188 189 int 190 mtsleep(wchan_t ident, pri_t priority, const char *wmesg, int timo, 191 kmutex_t *mtx) 192 { 193 struct lwp *l = curlwp; 194 sleepq_t *sq; 195 int error; 196 197 if (sleepq_dontsleep(l)) { 198 (void)sleepq_abort(mtx, (priority & PNORELOCK) != 0); 199 return 0; 200 } 201 202 sq = sleeptab_lookup(&sleeptab, ident); 203 sleepq_enter(sq, l); 204 sleepq_enqueue(sq, priority & PRIMASK, ident, wmesg, &sleep_syncobj); 205 mutex_exit(mtx); 206 error = sleepq_block(timo, priority & PCATCH); 207 208 if ((priority & PNORELOCK) == 0) 209 mutex_enter(mtx); 210 211 return error; 212 } 213 214 /* 215 * General sleep call for situations where a wake-up is not expected. 216 */ 217 int 218 kpause(const char *wmesg, bool intr, int timo, kmutex_t *mtx) 219 { 220 struct lwp *l = curlwp; 221 sleepq_t *sq; 222 int error; 223 224 if (sleepq_dontsleep(l)) 225 return sleepq_abort(NULL, 0); 226 227 if (mtx != NULL) 228 mutex_exit(mtx); 229 sq = sleeptab_lookup(&sleeptab, l); 230 sleepq_enter(sq, l); 231 sleepq_enqueue(sq, sched_kpri(l), l, wmesg, &sleep_syncobj); 232 error = sleepq_block(timo, intr); 233 if (mtx != NULL) 234 mutex_enter(mtx); 235 236 return error; 237 } 238 239 /* 240 * OBSOLETE INTERFACE 241 * 242 * Make all processes sleeping on the specified identifier runnable. 243 */ 244 void 245 wakeup(wchan_t ident) 246 { 247 sleepq_t *sq; 248 249 if (cold) 250 return; 251 252 sq = sleeptab_lookup(&sleeptab, ident); 253 sleepq_wake(sq, ident, (u_int)-1); 254 } 255 256 /* 257 * OBSOLETE INTERFACE 258 * 259 * Make the highest priority process first in line on the specified 260 * identifier runnable. 261 */ 262 void 263 wakeup_one(wchan_t ident) 264 { 265 sleepq_t *sq; 266 267 if (cold) 268 return; 269 270 sq = sleeptab_lookup(&sleeptab, ident); 271 sleepq_wake(sq, ident, 1); 272 } 273 274 275 /* 276 * General yield call. Puts the current process back on its run queue and 277 * performs a voluntary context switch. Should only be called when the 278 * current process explicitly requests it (eg sched_yield(2) in compat code). 279 */ 280 void 281 yield(void) 282 { 283 struct lwp *l = curlwp; 284 285 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 286 lwp_lock(l); 287 KASSERT(lwp_locked(l, &l->l_cpu->ci_schedstate.spc_lwplock)); 288 KASSERT(l->l_stat == LSONPROC); 289 l->l_priority = l->l_usrpri; 290 (void)mi_switch(l); 291 KERNEL_LOCK(l->l_biglocks, l); 292 } 293 294 /* 295 * General preemption call. Puts the current process back on its run queue 296 * and performs an involuntary context switch. 297 */ 298 void 299 preempt(void) 300 { 301 struct lwp *l = curlwp; 302 303 KERNEL_UNLOCK_ALL(l, &l->l_biglocks); 304 lwp_lock(l); 305 KASSERT(lwp_locked(l, &l->l_cpu->ci_schedstate.spc_lwplock)); 306 KASSERT(l->l_stat == LSONPROC); 307 l->l_priority = l->l_usrpri; 308 l->l_nivcsw++; 309 (void)mi_switch(l); 310 KERNEL_LOCK(l->l_biglocks, l); 311 } 312 313 /* 314 * Compute the amount of time during which the current lwp was running. 315 * 316 * - update l_rtime unless it's an idle lwp. 317 * - update spc_runtime for the next lwp. 318 */ 319 320 static inline void 321 updatertime(struct lwp *l, struct schedstate_percpu *spc) 322 { 323 struct timeval tv; 324 long s, u; 325 326 if ((l->l_flag & LW_IDLE) != 0) { 327 microtime(&spc->spc_runtime); 328 return; 329 } 330 331 microtime(&tv); 332 u = l->l_rtime.tv_usec + (tv.tv_usec - spc->spc_runtime.tv_usec); 333 s = l->l_rtime.tv_sec + (tv.tv_sec - spc->spc_runtime.tv_sec); 334 if (u < 0) { 335 u += 1000000; 336 s--; 337 } else if (u >= 1000000) { 338 u -= 1000000; 339 s++; 340 } 341 l->l_rtime.tv_usec = u; 342 l->l_rtime.tv_sec = s; 343 344 spc->spc_runtime = tv; 345 } 346 347 /* 348 * The machine independent parts of context switch. 349 * 350 * Returns 1 if another LWP was actually run. 351 */ 352 int 353 mi_switch(struct lwp *l) 354 { 355 struct schedstate_percpu *spc; 356 struct lwp *newl; 357 int retval, oldspl; 358 359 KASSERT(lwp_locked(l, NULL)); 360 LOCKDEBUG_BARRIER(l->l_mutex, 1); 361 362 #ifdef KSTACK_CHECK_MAGIC 363 kstack_check_magic(l); 364 #endif 365 366 /* 367 * It's safe to read the per CPU schedstate unlocked here, as all we 368 * are after is the run time and that's guarenteed to have been last 369 * updated by this CPU. 370 */ 371 KDASSERT(l->l_cpu == curcpu()); 372 373 /* 374 * Process is about to yield the CPU; clear the appropriate 375 * scheduling flags. 376 */ 377 spc = &l->l_cpu->ci_schedstate; 378 newl = NULL; 379 380 if (l->l_switchto != NULL) { 381 newl = l->l_switchto; 382 l->l_switchto = NULL; 383 } 384 385 /* Count time spent in current system call */ 386 SYSCALL_TIME_SLEEP(l); 387 388 /* 389 * XXXSMP If we are using h/w performance counters, 390 * save context. 391 */ 392 #if PERFCTRS 393 if (PMC_ENABLED(l->l_proc)) { 394 pmc_save_context(l->l_proc); 395 } 396 #endif 397 updatertime(l, spc); 398 399 /* 400 * If on the CPU and we have gotten this far, then we must yield. 401 */ 402 mutex_spin_enter(spc->spc_mutex); 403 spc->spc_flags &= ~SPCF_SWITCHCLEAR; 404 KASSERT(l->l_stat != LSRUN); 405 if (l->l_stat == LSONPROC) { 406 KASSERT(lwp_locked(l, &spc->spc_lwplock)); 407 if ((l->l_flag & LW_IDLE) == 0) { 408 l->l_stat = LSRUN; 409 lwp_setlock(l, spc->spc_mutex); 410 sched_enqueue(l, true); 411 } else 412 l->l_stat = LSIDL; 413 } 414 415 /* 416 * Let sched_nextlwp() select the LWP to run the CPU next. 417 * If no LWP is runnable, switch to the idle LWP. 418 */ 419 if (newl == NULL) { 420 newl = sched_nextlwp(); 421 if (newl != NULL) { 422 sched_dequeue(newl); 423 KASSERT(lwp_locked(newl, spc->spc_mutex)); 424 newl->l_stat = LSONPROC; 425 newl->l_cpu = l->l_cpu; 426 newl->l_flag |= LW_RUNNING; 427 lwp_setlock(newl, &spc->spc_lwplock); 428 } else { 429 newl = l->l_cpu->ci_data.cpu_idlelwp; 430 newl->l_stat = LSONPROC; 431 newl->l_flag |= LW_RUNNING; 432 } 433 spc->spc_curpriority = newl->l_usrpri; 434 newl->l_priority = newl->l_usrpri; 435 cpu_did_resched(); 436 } 437 438 if (l != newl) { 439 struct lwp *prevlwp; 440 441 /* 442 * If the old LWP has been moved to a run queue above, 443 * drop the general purpose LWP lock: it's now locked 444 * by the scheduler lock. 445 * 446 * Otherwise, drop the scheduler lock. We're done with 447 * the run queues for now. 448 */ 449 if (l->l_mutex == spc->spc_mutex) { 450 mutex_spin_exit(&spc->spc_lwplock); 451 } else { 452 mutex_spin_exit(spc->spc_mutex); 453 } 454 455 /* Unlocked, but for statistics only. */ 456 uvmexp.swtch++; 457 458 /* Save old VM context. */ 459 pmap_deactivate(l); 460 461 /* Switch to the new LWP.. */ 462 l->l_ncsw++; 463 l->l_flag &= ~LW_RUNNING; 464 oldspl = MUTEX_SPIN_OLDSPL(l->l_cpu); 465 prevlwp = cpu_switchto(l, newl); 466 467 /* 468 * .. we have switched away and are now back so we must 469 * be the new curlwp. prevlwp is who we replaced. 470 */ 471 curlwp = l; 472 if (prevlwp != NULL) { 473 curcpu()->ci_mtx_oldspl = oldspl; 474 lwp_unlock(prevlwp); 475 } else { 476 splx(oldspl); 477 } 478 479 /* Restore VM context. */ 480 pmap_activate(l); 481 retval = 1; 482 } else { 483 /* Nothing to do - just unlock and return. */ 484 mutex_spin_exit(spc->spc_mutex); 485 lwp_unlock(l); 486 retval = 0; 487 } 488 489 KASSERT(l == curlwp); 490 KASSERT(l->l_stat == LSONPROC); 491 492 /* 493 * XXXSMP If we are using h/w performance counters, restore context. 494 */ 495 #if PERFCTRS 496 if (PMC_ENABLED(l->l_proc)) { 497 pmc_restore_context(l->l_proc); 498 } 499 #endif 500 501 /* 502 * We're running again; record our new start time. We might 503 * be running on a new CPU now, so don't use the cached 504 * schedstate_percpu pointer. 505 */ 506 SYSCALL_TIME_WAKEUP(l); 507 KDASSERT(l->l_cpu == curcpu()); 508 LOCKDEBUG_BARRIER(NULL, 1); 509 510 return retval; 511 } 512 513 /* 514 * Change process state to be runnable, placing it on the run queue if it is 515 * in memory, and awakening the swapper if it isn't in memory. 516 * 517 * Call with the process and LWP locked. Will return with the LWP unlocked. 518 */ 519 void 520 setrunnable(struct lwp *l) 521 { 522 struct proc *p = l->l_proc; 523 sigset_t *ss; 524 525 KASSERT((l->l_flag & LW_IDLE) == 0); 526 KASSERT(mutex_owned(&p->p_smutex)); 527 KASSERT(lwp_locked(l, NULL)); 528 529 switch (l->l_stat) { 530 case LSSTOP: 531 /* 532 * If we're being traced (possibly because someone attached us 533 * while we were stopped), check for a signal from the debugger. 534 */ 535 if ((p->p_slflag & PSL_TRACED) != 0 && p->p_xstat != 0) { 536 if ((sigprop[p->p_xstat] & SA_TOLWP) != 0) 537 ss = &l->l_sigpend.sp_set; 538 else 539 ss = &p->p_sigpend.sp_set; 540 sigaddset(ss, p->p_xstat); 541 signotify(l); 542 } 543 p->p_nrlwps++; 544 break; 545 case LSSUSPENDED: 546 l->l_flag &= ~LW_WSUSPEND; 547 p->p_nrlwps++; 548 cv_broadcast(&p->p_lwpcv); 549 break; 550 case LSSLEEP: 551 KASSERT(l->l_wchan != NULL); 552 break; 553 default: 554 panic("setrunnable: lwp %p state was %d", l, l->l_stat); 555 } 556 557 /* 558 * If the LWP was sleeping interruptably, then it's OK to start it 559 * again. If not, mark it as still sleeping. 560 */ 561 if (l->l_wchan != NULL) { 562 l->l_stat = LSSLEEP; 563 /* lwp_unsleep() will release the lock. */ 564 lwp_unsleep(l); 565 return; 566 } 567 568 /* 569 * If the LWP is still on the CPU, mark it as LSONPROC. It may be 570 * about to call mi_switch(), in which case it will yield. 571 */ 572 if ((l->l_flag & LW_RUNNING) != 0) { 573 l->l_stat = LSONPROC; 574 l->l_slptime = 0; 575 lwp_unlock(l); 576 return; 577 } 578 579 /* 580 * Set the LWP runnable. If it's swapped out, we need to wake the swapper 581 * to bring it back in. Otherwise, enter it into a run queue. 582 */ 583 if (l->l_mutex != l->l_cpu->ci_schedstate.spc_mutex) { 584 spc_lock(l->l_cpu); 585 lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_mutex); 586 } 587 588 sched_setrunnable(l); 589 l->l_stat = LSRUN; 590 l->l_slptime = 0; 591 592 if (l->l_flag & LW_INMEM) { 593 sched_enqueue(l, false); 594 resched_cpu(l); 595 lwp_unlock(l); 596 } else { 597 lwp_unlock(l); 598 uvm_kick_scheduler(); 599 } 600 } 601 602 /* 603 * suspendsched: 604 * 605 * Convert all non-L_SYSTEM LSSLEEP or LSRUN LWPs to LSSUSPENDED. 606 */ 607 void 608 suspendsched(void) 609 { 610 CPU_INFO_ITERATOR cii; 611 struct cpu_info *ci; 612 struct lwp *l; 613 struct proc *p; 614 615 /* 616 * We do this by process in order not to violate the locking rules. 617 */ 618 mutex_enter(&proclist_mutex); 619 PROCLIST_FOREACH(p, &allproc) { 620 mutex_enter(&p->p_smutex); 621 622 if ((p->p_flag & PK_SYSTEM) != 0) { 623 mutex_exit(&p->p_smutex); 624 continue; 625 } 626 627 p->p_stat = SSTOP; 628 629 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 630 if (l == curlwp) 631 continue; 632 633 lwp_lock(l); 634 635 /* 636 * Set L_WREBOOT so that the LWP will suspend itself 637 * when it tries to return to user mode. We want to 638 * try and get to get as many LWPs as possible to 639 * the user / kernel boundary, so that they will 640 * release any locks that they hold. 641 */ 642 l->l_flag |= (LW_WREBOOT | LW_WSUSPEND); 643 644 if (l->l_stat == LSSLEEP && 645 (l->l_flag & LW_SINTR) != 0) { 646 /* setrunnable() will release the lock. */ 647 setrunnable(l); 648 continue; 649 } 650 651 lwp_unlock(l); 652 } 653 654 mutex_exit(&p->p_smutex); 655 } 656 mutex_exit(&proclist_mutex); 657 658 /* 659 * Kick all CPUs to make them preempt any LWPs running in user mode. 660 * They'll trap into the kernel and suspend themselves in userret(). 661 */ 662 for (CPU_INFO_FOREACH(cii, ci)) 663 cpu_need_resched(ci, 0); 664 } 665 666 /* 667 * sched_kpri: 668 * 669 * Scale a priority level to a kernel priority level, usually 670 * for an LWP that is about to sleep. 671 */ 672 pri_t 673 sched_kpri(struct lwp *l) 674 { 675 /* 676 * Scale user priorities (127 -> 50) up to kernel priorities 677 * in the range (49 -> 8). Reserve the top 8 kernel priorities 678 * for high priority kthreads. Kernel priorities passed in 679 * are left "as is". XXX This is somewhat arbitrary. 680 */ 681 static const uint8_t kpri_tab[] = { 682 0, 1, 2, 3, 4, 5, 6, 7, 683 8, 9, 10, 11, 12, 13, 14, 15, 684 16, 17, 18, 19, 20, 21, 22, 23, 685 24, 25, 26, 27, 28, 29, 30, 31, 686 32, 33, 34, 35, 36, 37, 38, 39, 687 40, 41, 42, 43, 44, 45, 46, 47, 688 48, 49, 8, 8, 9, 9, 10, 10, 689 11, 11, 12, 12, 13, 14, 14, 15, 690 15, 16, 16, 17, 17, 18, 18, 19, 691 20, 20, 21, 21, 22, 22, 23, 23, 692 24, 24, 25, 26, 26, 27, 27, 28, 693 28, 29, 29, 30, 30, 31, 32, 32, 694 33, 33, 34, 34, 35, 35, 36, 36, 695 37, 38, 38, 39, 39, 40, 40, 41, 696 41, 42, 42, 43, 44, 44, 45, 45, 697 46, 46, 47, 47, 48, 48, 49, 49, 698 }; 699 700 return (pri_t)kpri_tab[l->l_usrpri]; 701 } 702 703 /* 704 * sched_unsleep: 705 * 706 * The is called when the LWP has not been awoken normally but instead 707 * interrupted: for example, if the sleep timed out. Because of this, 708 * it's not a valid action for running or idle LWPs. 709 */ 710 static void 711 sched_unsleep(struct lwp *l) 712 { 713 714 lwp_unlock(l); 715 panic("sched_unsleep"); 716 } 717 718 inline void 719 resched_cpu(struct lwp *l) 720 { 721 struct cpu_info *ci; 722 const pri_t pri = lwp_eprio(l); 723 724 /* 725 * XXXSMP 726 * Since l->l_cpu persists across a context switch, 727 * this gives us *very weak* processor affinity, in 728 * that we notify the CPU on which the process last 729 * ran that it should try to switch. 730 * 731 * This does not guarantee that the process will run on 732 * that processor next, because another processor might 733 * grab it the next time it performs a context switch. 734 * 735 * This also does not handle the case where its last 736 * CPU is running a higher-priority process, but every 737 * other CPU is running a lower-priority process. There 738 * are ways to handle this situation, but they're not 739 * currently very pretty, and we also need to weigh the 740 * cost of moving a process from one CPU to another. 741 */ 742 ci = (l->l_cpu != NULL) ? l->l_cpu : curcpu(); 743 if (pri < ci->ci_schedstate.spc_curpriority) 744 cpu_need_resched(ci, 0); 745 } 746 747 static void 748 sched_changepri(struct lwp *l, pri_t pri) 749 { 750 751 KASSERT(lwp_locked(l, NULL)); 752 753 l->l_usrpri = pri; 754 if (l->l_priority < PUSER) 755 return; 756 757 if (l->l_stat != LSRUN || (l->l_flag & LW_INMEM) == 0) { 758 l->l_priority = pri; 759 return; 760 } 761 762 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 763 764 sched_dequeue(l); 765 l->l_priority = pri; 766 sched_enqueue(l, false); 767 resched_cpu(l); 768 } 769 770 static void 771 sched_lendpri(struct lwp *l, pri_t pri) 772 { 773 774 KASSERT(lwp_locked(l, NULL)); 775 776 if (l->l_stat != LSRUN || (l->l_flag & LW_INMEM) == 0) { 777 l->l_inheritedprio = pri; 778 return; 779 } 780 781 KASSERT(lwp_locked(l, l->l_cpu->ci_schedstate.spc_mutex)); 782 783 sched_dequeue(l); 784 l->l_inheritedprio = pri; 785 sched_enqueue(l, false); 786 resched_cpu(l); 787 } 788 789 struct lwp * 790 syncobj_noowner(wchan_t wchan) 791 { 792 793 return NULL; 794 } 795 796 797 /* decay 95% of `p_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ 798 fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ 799 800 /* 801 * If `ccpu' is not equal to `exp(-1/20)' and you still want to use the 802 * faster/more-accurate formula, you'll have to estimate CCPU_SHIFT below 803 * and possibly adjust FSHIFT in "param.h" so that (FSHIFT >= CCPU_SHIFT). 804 * 805 * To estimate CCPU_SHIFT for exp(-1/20), the following formula was used: 806 * 1 - exp(-1/20) ~= 0.0487 ~= 0.0488 == 1 (fixed pt, *11* bits). 807 * 808 * If you dont want to bother with the faster/more-accurate formula, you 809 * can set CCPU_SHIFT to (FSHIFT + 1) which will use a slower/less-accurate 810 * (more general) method of calculating the %age of CPU used by a process. 811 */ 812 #define CCPU_SHIFT (FSHIFT + 1) 813 814 /* 815 * sched_pstats: 816 * 817 * Update process statistics and check CPU resource allocation. 818 * Call scheduler-specific hook to eventually adjust process/LWP 819 * priorities. 820 * 821 * XXXSMP This needs to be reorganised in order to reduce the locking 822 * burden. 823 */ 824 /* ARGSUSED */ 825 void 826 sched_pstats(void *arg) 827 { 828 struct rlimit *rlim; 829 struct lwp *l; 830 struct proc *p; 831 int minslp, sig, clkhz; 832 long runtm; 833 834 sched_pstats_ticks++; 835 836 mutex_enter(&proclist_mutex); 837 PROCLIST_FOREACH(p, &allproc) { 838 /* 839 * Increment time in/out of memory and sleep time (if 840 * sleeping). We ignore overflow; with 16-bit int's 841 * (remember them?) overflow takes 45 days. 842 */ 843 minslp = 2; 844 mutex_enter(&p->p_smutex); 845 mutex_spin_enter(&p->p_stmutex); 846 runtm = p->p_rtime.tv_sec; 847 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 848 if ((l->l_flag & LW_IDLE) != 0) 849 continue; 850 lwp_lock(l); 851 runtm += l->l_rtime.tv_sec; 852 l->l_swtime++; 853 if (l->l_stat == LSSLEEP || l->l_stat == LSSTOP || 854 l->l_stat == LSSUSPENDED) { 855 l->l_slptime++; 856 minslp = min(minslp, l->l_slptime); 857 } else 858 minslp = 0; 859 lwp_unlock(l); 860 861 /* 862 * p_pctcpu is only for ps. 863 */ 864 l->l_pctcpu = (l->l_pctcpu * ccpu) >> FSHIFT; 865 if (l->l_slptime < 1) { 866 clkhz = stathz != 0 ? stathz : hz; 867 #if (FSHIFT >= CCPU_SHIFT) 868 l->l_pctcpu += (clkhz == 100) ? 869 ((fixpt_t)l->l_cpticks) << 870 (FSHIFT - CCPU_SHIFT) : 871 100 * (((fixpt_t) p->p_cpticks) 872 << (FSHIFT - CCPU_SHIFT)) / clkhz; 873 #else 874 l->l_pctcpu += ((FSCALE - ccpu) * 875 (l->l_cpticks * FSCALE / clkhz)) >> FSHIFT; 876 #endif 877 l->l_cpticks = 0; 878 } 879 } 880 p->p_pctcpu = (p->p_pctcpu * ccpu) >> FSHIFT; 881 sched_pstats_hook(p, minslp); 882 mutex_spin_exit(&p->p_stmutex); 883 884 /* 885 * Check if the process exceeds its CPU resource allocation. 886 * If over max, kill it. 887 */ 888 rlim = &p->p_rlimit[RLIMIT_CPU]; 889 sig = 0; 890 if (runtm >= rlim->rlim_cur) { 891 if (runtm >= rlim->rlim_max) 892 sig = SIGKILL; 893 else { 894 sig = SIGXCPU; 895 if (rlim->rlim_cur < rlim->rlim_max) 896 rlim->rlim_cur += 5; 897 } 898 } 899 mutex_exit(&p->p_smutex); 900 if (sig) { 901 psignal(p, sig); 902 } 903 } 904 mutex_exit(&proclist_mutex); 905 uvm_meter(); 906 cv_wakeup(&lbolt); 907 callout_schedule(&sched_pstats_ch, hz); 908 } 909 910 void 911 sched_init(void) 912 { 913 914 cv_init(&lbolt, "lbolt"); 915 callout_init(&sched_pstats_ch, 0); 916 callout_setfunc(&sched_pstats_ch, sched_pstats, NULL); 917 sched_setup(); 918 sched_pstats(NULL); 919 } 920