1 /* $OpenBSD: kern_synch.c,v 1.213 2024/11/11 13:28:29 claudio Exp $ */ 2 /* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1990, 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * (c) UNIX System Laboratories, Inc. 8 * All or some portions of this file are derived from material licensed 9 * to the University of California by American Telephone and Telegraph 10 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 11 * the permission of UNIX System Laboratories, Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * @(#)kern_synch.c 8.6 (Berkeley) 1/21/94 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/proc.h> 43 #include <sys/kernel.h> 44 #include <sys/signalvar.h> 45 #include <sys/sched.h> 46 #include <sys/timeout.h> 47 #include <sys/mount.h> 48 #include <sys/syscallargs.h> 49 #include <sys/refcnt.h> 50 #include <sys/atomic.h> 51 #include <sys/tracepoint.h> 52 53 #include <ddb/db_output.h> 54 55 #include <machine/spinlock.h> 56 57 #ifdef DIAGNOSTIC 58 #include <sys/syslog.h> 59 #endif 60 61 #ifdef KTRACE 62 #include <sys/ktrace.h> 63 #endif 64 65 int sleep_signal_check(struct proc *, int); 66 int thrsleep(struct proc *, struct sys___thrsleep_args *); 67 int thrsleep_unlock(void *); 68 69 extern void proc_stop(struct proc *p, int); 70 71 /* 72 * We're only looking at 7 bits of the address; everything is 73 * aligned to 4, lots of things are aligned to greater powers 74 * of 2. Shift right by 8, i.e. drop the bottom 256 worth. 75 */ 76 #define TABLESIZE 128 77 #define LOOKUP(x) (((long)(x) >> 8) & (TABLESIZE - 1)) 78 TAILQ_HEAD(slpque,proc) slpque[TABLESIZE]; 79 80 void 81 sleep_queue_init(void) 82 { 83 int i; 84 85 for (i = 0; i < TABLESIZE; i++) 86 TAILQ_INIT(&slpque[i]); 87 } 88 89 /* 90 * Global sleep channel for threads that do not want to 91 * receive wakeup(9) broadcasts. 92 */ 93 int nowake; 94 95 /* 96 * During autoconfiguration or after a panic, a sleep will simply 97 * lower the priority briefly to allow interrupts, then return. 98 * The priority to be used (safepri) is machine-dependent, thus this 99 * value is initialized and maintained in the machine-dependent layers. 100 * This priority will typically be 0, or the lowest priority 101 * that is safe for use on the interrupt stack; it can be made 102 * higher to block network software interrupts after panics. 103 */ 104 extern int safepri; 105 106 /* 107 * General sleep call. Suspends the current process until a wakeup is 108 * performed on the specified identifier. The process will then be made 109 * runnable with the specified priority. Sleeps at most timo/hz seconds 110 * (0 means no timeout). If pri includes PCATCH flag, signals are checked 111 * before and after sleeping, else signals are not checked. Returns 0 if 112 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 113 * signal needs to be delivered, ERESTART is returned if the current system 114 * call should be restarted if possible, and EINTR is returned if the system 115 * call should be interrupted by the signal (return EINTR). 116 */ 117 int 118 tsleep(const volatile void *ident, int priority, const char *wmesg, int timo) 119 { 120 #ifdef MULTIPROCESSOR 121 int hold_count; 122 #endif 123 124 KASSERT((priority & ~(PRIMASK | PCATCH)) == 0); 125 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 126 127 #ifdef MULTIPROCESSOR 128 KASSERT(ident == &nowake || timo || _kernel_lock_held()); 129 #endif 130 131 #ifdef DDB 132 if (cold == 2) 133 db_stack_dump(); 134 #endif 135 if (cold || panicstr) { 136 int s; 137 /* 138 * After a panic, or during autoconfiguration, 139 * just give interrupts a chance, then just return; 140 * don't run any other procs or panic below, 141 * in case this is the idle process and already asleep. 142 */ 143 s = splhigh(); 144 splx(safepri); 145 #ifdef MULTIPROCESSOR 146 if (_kernel_lock_held()) { 147 hold_count = __mp_release_all(&kernel_lock); 148 __mp_acquire_count(&kernel_lock, hold_count); 149 } 150 #endif 151 splx(s); 152 return (0); 153 } 154 155 sleep_setup(ident, priority, wmesg); 156 return sleep_finish(timo, 1); 157 } 158 159 int 160 tsleep_nsec(const volatile void *ident, int priority, const char *wmesg, 161 uint64_t nsecs) 162 { 163 uint64_t to_ticks; 164 165 if (nsecs == INFSLP) 166 return tsleep(ident, priority, wmesg, 0); 167 #ifdef DIAGNOSTIC 168 if (nsecs == 0) { 169 log(LOG_WARNING, 170 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 171 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 172 wmesg); 173 } 174 #endif 175 /* 176 * We want to sleep at least nsecs nanoseconds worth of ticks. 177 * 178 * - Clamp nsecs to prevent arithmetic overflow. 179 * 180 * - Round nsecs up to account for any nanoseconds that do not 181 * divide evenly into tick_nsec, otherwise we'll lose them to 182 * integer division in the next step. We add (tick_nsec - 1) 183 * to keep from introducing a spurious tick if there are no 184 * such nanoseconds, i.e. nsecs % tick_nsec == 0. 185 * 186 * - Divide the rounded value to a count of ticks. We divide 187 * by (tick_nsec + 1) to discard the extra tick introduced if, 188 * before rounding, nsecs % tick_nsec == 1. 189 * 190 * - Finally, add a tick to the result. We need to wait out 191 * the current tick before we can begin counting our interval, 192 * as we do not know how much time has elapsed since the 193 * current tick began. 194 */ 195 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 196 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 197 if (to_ticks > INT_MAX) 198 to_ticks = INT_MAX; 199 return tsleep(ident, priority, wmesg, (int)to_ticks); 200 } 201 202 /* 203 * Same as tsleep, but if we have a mutex provided, then once we've 204 * entered the sleep queue we drop the mutex. After sleeping we re-lock. 205 */ 206 int 207 msleep(const volatile void *ident, struct mutex *mtx, int priority, 208 const char *wmesg, int timo) 209 { 210 int error, spl; 211 #ifdef MULTIPROCESSOR 212 int hold_count; 213 #endif 214 215 KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); 216 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 217 KASSERT(mtx != NULL); 218 219 #ifdef DDB 220 if (cold == 2) 221 db_stack_dump(); 222 #endif 223 if (cold || panicstr) { 224 /* 225 * After a panic, or during autoconfiguration, 226 * just give interrupts a chance, then just return; 227 * don't run any other procs or panic below, 228 * in case this is the idle process and already asleep. 229 */ 230 spl = MUTEX_OLDIPL(mtx); 231 MUTEX_OLDIPL(mtx) = safepri; 232 mtx_leave(mtx); 233 #ifdef MULTIPROCESSOR 234 if (_kernel_lock_held()) { 235 hold_count = __mp_release_all(&kernel_lock); 236 __mp_acquire_count(&kernel_lock, hold_count); 237 } 238 #endif 239 if ((priority & PNORELOCK) == 0) { 240 mtx_enter(mtx); 241 MUTEX_OLDIPL(mtx) = spl; 242 } else 243 splx(spl); 244 return (0); 245 } 246 247 sleep_setup(ident, priority, wmesg); 248 249 mtx_leave(mtx); 250 /* signal may stop the process, release mutex before that */ 251 error = sleep_finish(timo, 1); 252 253 if ((priority & PNORELOCK) == 0) 254 mtx_enter(mtx); 255 256 return error; 257 } 258 259 int 260 msleep_nsec(const volatile void *ident, struct mutex *mtx, int priority, 261 const char *wmesg, uint64_t nsecs) 262 { 263 uint64_t to_ticks; 264 265 if (nsecs == INFSLP) 266 return msleep(ident, mtx, priority, wmesg, 0); 267 #ifdef DIAGNOSTIC 268 if (nsecs == 0) { 269 log(LOG_WARNING, 270 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 271 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 272 wmesg); 273 } 274 #endif 275 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 276 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 277 if (to_ticks > INT_MAX) 278 to_ticks = INT_MAX; 279 return msleep(ident, mtx, priority, wmesg, (int)to_ticks); 280 } 281 282 /* 283 * Same as tsleep, but if we have a rwlock provided, then once we've 284 * entered the sleep queue we drop the it. After sleeping we re-lock. 285 */ 286 int 287 rwsleep(const volatile void *ident, struct rwlock *rwl, int priority, 288 const char *wmesg, int timo) 289 { 290 int error, status; 291 292 KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); 293 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 294 KASSERT(ident != rwl); 295 rw_assert_anylock(rwl); 296 status = rw_status(rwl); 297 298 sleep_setup(ident, priority, wmesg); 299 300 rw_exit(rwl); 301 /* signal may stop the process, release rwlock before that */ 302 error = sleep_finish(timo, 1); 303 304 if ((priority & PNORELOCK) == 0) 305 rw_enter(rwl, status); 306 307 return error; 308 } 309 310 int 311 rwsleep_nsec(const volatile void *ident, struct rwlock *rwl, int priority, 312 const char *wmesg, uint64_t nsecs) 313 { 314 uint64_t to_ticks; 315 316 if (nsecs == INFSLP) 317 return rwsleep(ident, rwl, priority, wmesg, 0); 318 #ifdef DIAGNOSTIC 319 if (nsecs == 0) { 320 log(LOG_WARNING, 321 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 322 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 323 wmesg); 324 } 325 #endif 326 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 327 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 328 if (to_ticks > INT_MAX) 329 to_ticks = INT_MAX; 330 return rwsleep(ident, rwl, priority, wmesg, (int)to_ticks); 331 } 332 333 void 334 sleep_setup(const volatile void *ident, int prio, const char *wmesg) 335 { 336 struct proc *p = curproc; 337 338 #ifdef DIAGNOSTIC 339 if (p->p_flag & P_CANTSLEEP) 340 panic("sleep: %s failed insomnia", p->p_p->ps_comm); 341 if (ident == NULL) 342 panic("sleep: no ident"); 343 if (p->p_stat != SONPROC) 344 panic("sleep: not SONPROC but %d", p->p_stat); 345 #endif 346 /* exiting processes are not allowed to catch signals */ 347 if (p->p_flag & P_WEXIT) 348 CLR(prio, PCATCH); 349 350 SCHED_LOCK(); 351 352 TRACEPOINT(sched, sleep, NULL); 353 354 p->p_wchan = ident; 355 p->p_wmesg = wmesg; 356 p->p_slptime = 0; 357 p->p_slppri = prio & PRIMASK; 358 atomic_setbits_int(&p->p_flag, P_WSLEEP); 359 TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], p, p_runq); 360 if (prio & PCATCH) 361 atomic_setbits_int(&p->p_flag, P_SINTR); 362 p->p_stat = SSLEEP; 363 364 SCHED_UNLOCK(); 365 } 366 367 int 368 sleep_finish(int timo, int do_sleep) 369 { 370 struct proc *p = curproc; 371 int catch, error = 0, error1 = 0; 372 373 catch = p->p_flag & P_SINTR; 374 375 if (timo != 0) { 376 KASSERT((p->p_flag & P_TIMEOUT) == 0); 377 timeout_add(&p->p_sleep_to, timo); 378 } 379 380 if (catch != 0) { 381 /* 382 * We put ourselves on the sleep queue and start our 383 * timeout before calling sleep_signal_check(), as we could 384 * stop there, and a wakeup or a SIGCONT (or both) could 385 * occur while we were stopped. A SIGCONT would cause 386 * us to be marked as SSLEEP without resuming us, thus 387 * we must be ready for sleep when sleep_signal_check() is 388 * called. 389 */ 390 if ((error = sleep_signal_check(p, 0)) != 0) { 391 catch = 0; 392 do_sleep = 0; 393 } 394 } 395 396 SCHED_LOCK(); 397 /* 398 * A few checks need to happen before going to sleep: 399 * - If the wakeup happens while going to sleep, p->p_wchan 400 * will be NULL. In that case unwind immediately but still 401 * check for possible signals and timeouts. 402 * - If the sleep is aborted call unsleep and take us of the 403 * sleep queue. 404 * - If requested to stop force a switch even if the sleep 405 * condition got cleared. 406 */ 407 if (p->p_wchan == NULL) 408 do_sleep = 0; 409 if (do_sleep == 0) 410 unsleep(p); 411 if (p->p_stat == SSTOP) 412 do_sleep = 1; 413 atomic_clearbits_int(&p->p_flag, P_WSLEEP); 414 415 if (do_sleep) { 416 KASSERT(p->p_stat == SSLEEP || p->p_stat == SSTOP); 417 p->p_ru.ru_nvcsw++; 418 mi_switch(); 419 } else { 420 KASSERT(p->p_stat == SONPROC || p->p_stat == SSLEEP); 421 p->p_stat = SONPROC; 422 } 423 424 #ifdef DIAGNOSTIC 425 if (p->p_stat != SONPROC) 426 panic("sleep_finish !SONPROC"); 427 #endif 428 429 p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri; 430 SCHED_UNLOCK(); 431 432 /* 433 * Even though this belongs to the signal handling part of sleep, 434 * we need to clear it before the ktrace. 435 */ 436 atomic_clearbits_int(&p->p_flag, P_SINTR); 437 438 if (timo != 0) { 439 if (p->p_flag & P_TIMEOUT) { 440 error1 = EWOULDBLOCK; 441 } else { 442 /* This can sleep. It must not use timeouts. */ 443 timeout_del_barrier(&p->p_sleep_to); 444 } 445 atomic_clearbits_int(&p->p_flag, P_TIMEOUT); 446 } 447 448 /* 449 * Check if thread was woken up because of a unwind or signal 450 * but ignore any pending stop condition. 451 */ 452 if (catch != 0) 453 error = sleep_signal_check(p, 1); 454 455 /* Signal errors are higher priority than timeouts. */ 456 if (error == 0 && error1 != 0) 457 error = error1; 458 459 return error; 460 } 461 462 /* 463 * Check and handle signals and suspensions around a sleep cycle. 464 * The 2nd call in sleep_finish() sets nostop = 1 and then stop 465 * signals can be ignored since the sleep is over and the process 466 * will stop in userret. 467 */ 468 int 469 sleep_signal_check(struct proc *p, int nostop) 470 { 471 struct sigctx ctx; 472 int err, sig; 473 474 if ((err = single_thread_check(p, 1)) != 0) 475 return err; 476 if ((sig = cursig(p, &ctx, 1)) != 0) { 477 if (ctx.sig_stop) { 478 if (nostop) 479 return 0; 480 p->p_p->ps_xsig = sig; 481 SCHED_LOCK(); 482 proc_stop(p, 0); 483 SCHED_UNLOCK(); 484 } else if (ctx.sig_intr) 485 return EINTR; 486 else 487 return ERESTART; 488 } 489 return 0; 490 } 491 492 int 493 wakeup_proc(struct proc *p, int flags) 494 { 495 int awakened = 0; 496 497 SCHED_ASSERT_LOCKED(); 498 499 if (p->p_wchan != NULL) { 500 awakened = 1; 501 if (flags) 502 atomic_setbits_int(&p->p_flag, flags); 503 #ifdef DIAGNOSTIC 504 if (p->p_stat != SSLEEP && p->p_stat != SSTOP) 505 panic("thread %d p_stat is %d", p->p_tid, p->p_stat); 506 #endif 507 unsleep(p); 508 if (p->p_stat == SSLEEP) 509 setrunnable(p); 510 } 511 512 return awakened; 513 } 514 515 516 /* 517 * Implement timeout for tsleep. 518 * If process hasn't been awakened (wchan non-zero), 519 * set timeout flag and undo the sleep. If proc 520 * is stopped, just unsleep so it will remain stopped. 521 */ 522 void 523 endtsleep(void *arg) 524 { 525 struct proc *p = arg; 526 527 SCHED_LOCK(); 528 wakeup_proc(p, P_TIMEOUT); 529 SCHED_UNLOCK(); 530 } 531 532 /* 533 * Remove a process from its wait queue 534 */ 535 void 536 unsleep(struct proc *p) 537 { 538 SCHED_ASSERT_LOCKED(); 539 540 if (p->p_wchan != NULL) { 541 TAILQ_REMOVE(&slpque[LOOKUP(p->p_wchan)], p, p_runq); 542 p->p_wchan = NULL; 543 p->p_wmesg = NULL; 544 TRACEPOINT(sched, unsleep, p->p_tid + THREAD_PID_OFFSET, 545 p->p_p->ps_pid); 546 } 547 } 548 549 /* 550 * Make a number of processes sleeping on the specified identifier runnable. 551 */ 552 void 553 wakeup_n(const volatile void *ident, int n) 554 { 555 struct slpque *qp, wakeq; 556 struct proc *p; 557 struct proc *pnext; 558 559 TAILQ_INIT(&wakeq); 560 561 SCHED_LOCK(); 562 qp = &slpque[LOOKUP(ident)]; 563 for (p = TAILQ_FIRST(qp); p != NULL && n != 0; p = pnext) { 564 pnext = TAILQ_NEXT(p, p_runq); 565 #ifdef DIAGNOSTIC 566 if (p->p_stat != SSLEEP && p->p_stat != SSTOP) 567 panic("thread %d p_stat is %d", p->p_tid, p->p_stat); 568 #endif 569 KASSERT(p->p_wchan != NULL); 570 if (p->p_wchan == ident) { 571 TAILQ_REMOVE(qp, p, p_runq); 572 p->p_wchan = NULL; 573 p->p_wmesg = NULL; 574 TAILQ_INSERT_TAIL(&wakeq, p, p_runq); 575 --n; 576 } 577 } 578 while ((p = TAILQ_FIRST(&wakeq))) { 579 TAILQ_REMOVE(&wakeq, p, p_runq); 580 TRACEPOINT(sched, unsleep, p->p_tid + THREAD_PID_OFFSET, 581 p->p_p->ps_pid); 582 if (p->p_stat == SSLEEP) 583 setrunnable(p); 584 } 585 SCHED_UNLOCK(); 586 } 587 588 /* 589 * Make all processes sleeping on the specified identifier runnable. 590 */ 591 void 592 wakeup(const volatile void *chan) 593 { 594 wakeup_n(chan, -1); 595 } 596 597 int 598 sys_sched_yield(struct proc *p, void *v, register_t *retval) 599 { 600 struct proc *q; 601 uint8_t newprio; 602 603 /* 604 * If one of the threads of a multi-threaded process called 605 * sched_yield(2), drop its priority to ensure its siblings 606 * can make some progress. 607 */ 608 mtx_enter(&p->p_p->ps_mtx); 609 newprio = p->p_usrpri; 610 TAILQ_FOREACH(q, &p->p_p->ps_threads, p_thr_link) 611 newprio = max(newprio, q->p_runpri); 612 mtx_leave(&p->p_p->ps_mtx); 613 614 SCHED_LOCK(); 615 setrunqueue(p->p_cpu, p, newprio); 616 p->p_ru.ru_nvcsw++; 617 mi_switch(); 618 SCHED_UNLOCK(); 619 620 return (0); 621 } 622 623 int 624 thrsleep_unlock(void *lock) 625 { 626 static _atomic_lock_t unlocked = _ATOMIC_LOCK_UNLOCKED; 627 _atomic_lock_t *atomiclock = lock; 628 629 if (!lock) 630 return 0; 631 632 return copyout(&unlocked, atomiclock, sizeof(unlocked)); 633 } 634 635 struct tslpentry { 636 TAILQ_ENTRY(tslpentry) tslp_link; 637 long tslp_ident; 638 }; 639 640 /* thrsleep queue shared between processes */ 641 static struct tslpqueue thrsleep_queue = TAILQ_HEAD_INITIALIZER(thrsleep_queue); 642 static struct rwlock thrsleep_lock = RWLOCK_INITIALIZER("thrsleeplk"); 643 644 int 645 thrsleep(struct proc *p, struct sys___thrsleep_args *v) 646 { 647 struct sys___thrsleep_args /* { 648 syscallarg(const volatile void *) ident; 649 syscallarg(clockid_t) clock_id; 650 syscallarg(const struct timespec *) tp; 651 syscallarg(void *) lock; 652 syscallarg(const int *) abort; 653 } */ *uap = v; 654 long ident = (long)SCARG(uap, ident); 655 struct tslpentry entry; 656 struct tslpqueue *queue; 657 struct rwlock *qlock; 658 struct timespec *tsp = (struct timespec *)SCARG(uap, tp); 659 void *lock = SCARG(uap, lock); 660 uint64_t nsecs = INFSLP; 661 int abort = 0, error; 662 clockid_t clock_id = SCARG(uap, clock_id); 663 664 if (ident == 0) 665 return (EINVAL); 666 if (tsp != NULL) { 667 struct timespec now; 668 669 if ((error = clock_gettime(p, clock_id, &now))) 670 return (error); 671 #ifdef KTRACE 672 if (KTRPOINT(p, KTR_STRUCT)) 673 ktrabstimespec(p, tsp); 674 #endif 675 676 if (timespeccmp(tsp, &now, <=)) { 677 /* already passed: still do the unlock */ 678 if ((error = thrsleep_unlock(lock))) 679 return (error); 680 return (EWOULDBLOCK); 681 } 682 683 timespecsub(tsp, &now, tsp); 684 nsecs = MIN(TIMESPEC_TO_NSEC(tsp), MAXTSLP); 685 } 686 687 if (ident == -1) { 688 queue = &thrsleep_queue; 689 qlock = &thrsleep_lock; 690 } else { 691 queue = &p->p_p->ps_tslpqueue; 692 qlock = &p->p_p->ps_lock; 693 } 694 695 /* Interlock with wakeup. */ 696 entry.tslp_ident = ident; 697 rw_enter_write(qlock); 698 TAILQ_INSERT_TAIL(queue, &entry, tslp_link); 699 rw_exit_write(qlock); 700 701 error = thrsleep_unlock(lock); 702 703 if (error == 0 && SCARG(uap, abort) != NULL) 704 error = copyin(SCARG(uap, abort), &abort, sizeof(abort)); 705 706 rw_enter_write(qlock); 707 if (error != 0) 708 goto out; 709 if (abort != 0) { 710 error = EINTR; 711 goto out; 712 } 713 if (entry.tslp_ident != 0) { 714 error = rwsleep_nsec(&entry, qlock, PWAIT|PCATCH, "thrsleep", 715 nsecs); 716 } 717 718 out: 719 if (entry.tslp_ident != 0) 720 TAILQ_REMOVE(queue, &entry, tslp_link); 721 rw_exit_write(qlock); 722 723 if (error == ERESTART) 724 error = ECANCELED; 725 726 return (error); 727 728 } 729 730 int 731 sys___thrsleep(struct proc *p, void *v, register_t *retval) 732 { 733 struct sys___thrsleep_args /* { 734 syscallarg(const volatile void *) ident; 735 syscallarg(clockid_t) clock_id; 736 syscallarg(struct timespec *) tp; 737 syscallarg(void *) lock; 738 syscallarg(const int *) abort; 739 } */ *uap = v; 740 struct timespec ts; 741 int error; 742 743 if (SCARG(uap, tp) != NULL) { 744 if ((error = copyin(SCARG(uap, tp), &ts, sizeof(ts)))) { 745 *retval = error; 746 return 0; 747 } 748 if (!timespecisvalid(&ts)) { 749 *retval = EINVAL; 750 return 0; 751 } 752 SCARG(uap, tp) = &ts; 753 } 754 755 *retval = thrsleep(p, uap); 756 return 0; 757 } 758 759 int 760 sys___thrwakeup(struct proc *p, void *v, register_t *retval) 761 { 762 struct sys___thrwakeup_args /* { 763 syscallarg(const volatile void *) ident; 764 syscallarg(int) n; 765 } */ *uap = v; 766 struct tslpentry *entry, *tmp; 767 struct tslpqueue *queue; 768 struct rwlock *qlock; 769 long ident = (long)SCARG(uap, ident); 770 int n = SCARG(uap, n); 771 int found = 0; 772 773 if (ident == 0) 774 *retval = EINVAL; 775 else { 776 if (ident == -1) { 777 queue = &thrsleep_queue; 778 qlock = &thrsleep_lock; 779 /* 780 * Wake up all waiters with ident -1. This is needed 781 * because ident -1 can be shared by multiple userspace 782 * lock state machines concurrently. The implementation 783 * has no way to direct the wakeup to a particular 784 * state machine. 785 */ 786 n = 0; 787 } else { 788 queue = &p->p_p->ps_tslpqueue; 789 qlock = &p->p_p->ps_lock; 790 } 791 792 rw_enter_write(qlock); 793 TAILQ_FOREACH_SAFE(entry, queue, tslp_link, tmp) { 794 if (entry->tslp_ident == ident) { 795 TAILQ_REMOVE(queue, entry, tslp_link); 796 entry->tslp_ident = 0; 797 wakeup_one(entry); 798 if (++found == n) 799 break; 800 } 801 } 802 rw_exit_write(qlock); 803 804 if (ident == -1) 805 *retval = 0; 806 else 807 *retval = found ? 0 : ESRCH; 808 } 809 810 return (0); 811 } 812 813 void 814 refcnt_init(struct refcnt *r) 815 { 816 refcnt_init_trace(r, 0); 817 } 818 819 void 820 refcnt_init_trace(struct refcnt *r, int idx) 821 { 822 r->r_traceidx = idx; 823 atomic_store_int(&r->r_refs, 1); 824 TRACEINDEX(refcnt, r->r_traceidx, r, 0, +1); 825 } 826 827 void 828 refcnt_take(struct refcnt *r) 829 { 830 u_int refs; 831 832 refs = atomic_inc_int_nv(&r->r_refs); 833 KASSERT(refs != 0); 834 TRACEINDEX(refcnt, r->r_traceidx, r, refs - 1, +1); 835 (void)refs; 836 } 837 838 int 839 refcnt_rele(struct refcnt *r) 840 { 841 u_int refs; 842 843 membar_exit_before_atomic(); 844 refs = atomic_dec_int_nv(&r->r_refs); 845 KASSERT(refs != ~0); 846 TRACEINDEX(refcnt, r->r_traceidx, r, refs + 1, -1); 847 if (refs == 0) { 848 membar_enter_after_atomic(); 849 return (1); 850 } 851 return (0); 852 } 853 854 void 855 refcnt_rele_wake(struct refcnt *r) 856 { 857 if (refcnt_rele(r)) 858 wakeup_one(r); 859 } 860 861 void 862 refcnt_finalize(struct refcnt *r, const char *wmesg) 863 { 864 u_int refs; 865 866 membar_exit_before_atomic(); 867 refs = atomic_dec_int_nv(&r->r_refs); 868 KASSERT(refs != ~0); 869 TRACEINDEX(refcnt, r->r_traceidx, r, refs + 1, -1); 870 while (refs) { 871 sleep_setup(r, PWAIT, wmesg); 872 refs = atomic_load_int(&r->r_refs); 873 sleep_finish(0, refs); 874 } 875 TRACEINDEX(refcnt, r->r_traceidx, r, refs, 0); 876 /* Order subsequent loads and stores after refs == 0 load. */ 877 membar_sync(); 878 } 879 880 int 881 refcnt_shared(struct refcnt *r) 882 { 883 u_int refs; 884 885 refs = atomic_load_int(&r->r_refs); 886 TRACEINDEX(refcnt, r->r_traceidx, r, refs, 0); 887 return (refs > 1); 888 } 889 890 unsigned int 891 refcnt_read(struct refcnt *r) 892 { 893 u_int refs; 894 895 refs = atomic_load_int(&r->r_refs); 896 TRACEINDEX(refcnt, r->r_traceidx, r, refs, 0); 897 return (refs); 898 } 899 900 void 901 cond_init(struct cond *c) 902 { 903 atomic_store_int(&c->c_wait, 1); 904 } 905 906 void 907 cond_signal(struct cond *c) 908 { 909 atomic_store_int(&c->c_wait, 0); 910 911 wakeup_one(c); 912 } 913 914 void 915 cond_wait(struct cond *c, const char *wmesg) 916 { 917 unsigned int wait; 918 919 wait = atomic_load_int(&c->c_wait); 920 while (wait) { 921 sleep_setup(c, PWAIT, wmesg); 922 wait = atomic_load_int(&c->c_wait); 923 sleep_finish(0, wait); 924 } 925 } 926