1 /* $OpenBSD: kern_synch.c,v 1.174 2021/01/11 13:55:53 claudio Exp $ */ 2 /* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1990, 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * (c) UNIX System Laboratories, Inc. 8 * All or some portions of this file are derived from material licensed 9 * to the University of California by American Telephone and Telegraph 10 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 11 * the permission of UNIX System Laboratories, Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * @(#)kern_synch.c 8.6 (Berkeley) 1/21/94 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/proc.h> 43 #include <sys/kernel.h> 44 #include <sys/signalvar.h> 45 #include <sys/resourcevar.h> 46 #include <sys/sched.h> 47 #include <sys/timeout.h> 48 #include <sys/mount.h> 49 #include <sys/syscallargs.h> 50 #include <sys/pool.h> 51 #include <sys/refcnt.h> 52 #include <sys/atomic.h> 53 #include <sys/smr.h> 54 #include <sys/witness.h> 55 #include <sys/tracepoint.h> 56 57 #include <ddb/db_output.h> 58 59 #include <machine/spinlock.h> 60 61 #ifdef DIAGNOSTIC 62 #include <sys/syslog.h> 63 #endif 64 65 #ifdef KTRACE 66 #include <sys/ktrace.h> 67 #endif 68 69 int sleep_signal_check(struct proc *); 70 int thrsleep(struct proc *, struct sys___thrsleep_args *); 71 int thrsleep_unlock(void *); 72 73 /* 74 * We're only looking at 7 bits of the address; everything is 75 * aligned to 4, lots of things are aligned to greater powers 76 * of 2. Shift right by 8, i.e. drop the bottom 256 worth. 77 */ 78 #define TABLESIZE 128 79 #define LOOKUP(x) (((long)(x) >> 8) & (TABLESIZE - 1)) 80 TAILQ_HEAD(slpque,proc) slpque[TABLESIZE]; 81 82 void 83 sleep_queue_init(void) 84 { 85 int i; 86 87 for (i = 0; i < TABLESIZE; i++) 88 TAILQ_INIT(&slpque[i]); 89 } 90 91 /* 92 * Global sleep channel for threads that do not want to 93 * receive wakeup(9) broadcasts. 94 */ 95 int nowake; 96 97 /* 98 * During autoconfiguration or after a panic, a sleep will simply 99 * lower the priority briefly to allow interrupts, then return. 100 * The priority to be used (safepri) is machine-dependent, thus this 101 * value is initialized and maintained in the machine-dependent layers. 102 * This priority will typically be 0, or the lowest priority 103 * that is safe for use on the interrupt stack; it can be made 104 * higher to block network software interrupts after panics. 105 */ 106 extern int safepri; 107 108 /* 109 * General sleep call. Suspends the current process until a wakeup is 110 * performed on the specified identifier. The process will then be made 111 * runnable with the specified priority. Sleeps at most timo/hz seconds 112 * (0 means no timeout). If pri includes PCATCH flag, signals are checked 113 * before and after sleeping, else signals are not checked. Returns 0 if 114 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 115 * signal needs to be delivered, ERESTART is returned if the current system 116 * call should be restarted if possible, and EINTR is returned if the system 117 * call should be interrupted by the signal (return EINTR). 118 */ 119 int 120 tsleep(const volatile void *ident, int priority, const char *wmesg, int timo) 121 { 122 struct sleep_state sls; 123 #ifdef MULTIPROCESSOR 124 int hold_count; 125 #endif 126 127 KASSERT((priority & ~(PRIMASK | PCATCH)) == 0); 128 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 129 130 #ifdef MULTIPROCESSOR 131 KASSERT(timo || _kernel_lock_held()); 132 #endif 133 134 #ifdef DDB 135 if (cold == 2) 136 db_stack_dump(); 137 #endif 138 if (cold || panicstr) { 139 int s; 140 /* 141 * After a panic, or during autoconfiguration, 142 * just give interrupts a chance, then just return; 143 * don't run any other procs or panic below, 144 * in case this is the idle process and already asleep. 145 */ 146 s = splhigh(); 147 splx(safepri); 148 #ifdef MULTIPROCESSOR 149 if (_kernel_lock_held()) { 150 hold_count = __mp_release_all(&kernel_lock); 151 __mp_acquire_count(&kernel_lock, hold_count); 152 } 153 #endif 154 splx(s); 155 return (0); 156 } 157 158 sleep_setup(&sls, ident, priority, wmesg); 159 sleep_setup_timeout(&sls, timo); 160 sleep_setup_signal(&sls); 161 162 return sleep_finish_all(&sls, 1); 163 } 164 165 int 166 tsleep_nsec(const volatile void *ident, int priority, const char *wmesg, 167 uint64_t nsecs) 168 { 169 uint64_t to_ticks; 170 171 if (nsecs == INFSLP) 172 return tsleep(ident, priority, wmesg, 0); 173 #ifdef DIAGNOSTIC 174 if (nsecs == 0) { 175 log(LOG_WARNING, 176 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 177 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 178 wmesg); 179 } 180 #endif 181 /* 182 * We want to sleep at least nsecs nanoseconds worth of ticks. 183 * 184 * - Clamp nsecs to prevent arithmetic overflow. 185 * 186 * - Round nsecs up to account for any nanoseconds that do not 187 * divide evenly into tick_nsec, otherwise we'll lose them to 188 * integer division in the next step. We add (tick_nsec - 1) 189 * to keep from introducing a spurious tick if there are no 190 * such nanoseconds, i.e. nsecs % tick_nsec == 0. 191 * 192 * - Divide the rounded value to a count of ticks. We divide 193 * by (tick_nsec + 1) to discard the extra tick introduced if, 194 * before rounding, nsecs % tick_nsec == 1. 195 * 196 * - Finally, add a tick to the result. We need to wait out 197 * the current tick before we can begin counting our interval, 198 * as we do not know how much time has elapsed since the 199 * current tick began. 200 */ 201 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 202 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 203 if (to_ticks > INT_MAX) 204 to_ticks = INT_MAX; 205 return tsleep(ident, priority, wmesg, (int)to_ticks); 206 } 207 208 /* 209 * Same as tsleep, but if we have a mutex provided, then once we've 210 * entered the sleep queue we drop the mutex. After sleeping we re-lock. 211 */ 212 int 213 msleep(const volatile void *ident, struct mutex *mtx, int priority, 214 const char *wmesg, int timo) 215 { 216 struct sleep_state sls; 217 int error, spl; 218 #ifdef MULTIPROCESSOR 219 int hold_count; 220 #endif 221 222 KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); 223 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 224 KASSERT(mtx != NULL); 225 226 if (priority & PCATCH) 227 KERNEL_ASSERT_LOCKED(); 228 229 if (cold || panicstr) { 230 /* 231 * After a panic, or during autoconfiguration, 232 * just give interrupts a chance, then just return; 233 * don't run any other procs or panic below, 234 * in case this is the idle process and already asleep. 235 */ 236 spl = MUTEX_OLDIPL(mtx); 237 MUTEX_OLDIPL(mtx) = safepri; 238 mtx_leave(mtx); 239 #ifdef MULTIPROCESSOR 240 if (_kernel_lock_held()) { 241 hold_count = __mp_release_all(&kernel_lock); 242 __mp_acquire_count(&kernel_lock, hold_count); 243 } 244 #endif 245 if ((priority & PNORELOCK) == 0) { 246 mtx_enter(mtx); 247 MUTEX_OLDIPL(mtx) = spl; 248 } else 249 splx(spl); 250 return (0); 251 } 252 253 sleep_setup(&sls, ident, priority, wmesg); 254 sleep_setup_timeout(&sls, timo); 255 256 /* XXX - We need to make sure that the mutex doesn't 257 * unblock splsched. This can be made a bit more 258 * correct when the sched_lock is a mutex. 259 */ 260 spl = MUTEX_OLDIPL(mtx); 261 MUTEX_OLDIPL(mtx) = splsched(); 262 mtx_leave(mtx); 263 /* signal may stop the process, release mutex before that */ 264 sleep_setup_signal(&sls); 265 266 error = sleep_finish_all(&sls, 1); 267 268 if ((priority & PNORELOCK) == 0) { 269 mtx_enter(mtx); 270 MUTEX_OLDIPL(mtx) = spl; /* put the ipl back */ 271 } else 272 splx(spl); 273 274 return error; 275 } 276 277 int 278 msleep_nsec(const volatile void *ident, struct mutex *mtx, int priority, 279 const char *wmesg, uint64_t nsecs) 280 { 281 uint64_t to_ticks; 282 283 if (nsecs == INFSLP) 284 return msleep(ident, mtx, priority, wmesg, 0); 285 #ifdef DIAGNOSTIC 286 if (nsecs == 0) { 287 log(LOG_WARNING, 288 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 289 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 290 wmesg); 291 } 292 #endif 293 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 294 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 295 if (to_ticks > INT_MAX) 296 to_ticks = INT_MAX; 297 return msleep(ident, mtx, priority, wmesg, (int)to_ticks); 298 } 299 300 /* 301 * Same as tsleep, but if we have a rwlock provided, then once we've 302 * entered the sleep queue we drop the it. After sleeping we re-lock. 303 */ 304 int 305 rwsleep(const volatile void *ident, struct rwlock *rwl, int priority, 306 const char *wmesg, int timo) 307 { 308 struct sleep_state sls; 309 int error, status; 310 311 KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); 312 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 313 rw_assert_anylock(rwl); 314 status = rw_status(rwl); 315 316 sleep_setup(&sls, ident, priority, wmesg); 317 sleep_setup_timeout(&sls, timo); 318 319 rw_exit(rwl); 320 /* signal may stop the process, release rwlock before that */ 321 sleep_setup_signal(&sls); 322 323 error = sleep_finish_all(&sls, 1); 324 325 if ((priority & PNORELOCK) == 0) 326 rw_enter(rwl, status); 327 328 return error; 329 } 330 331 int 332 rwsleep_nsec(const volatile void *ident, struct rwlock *rwl, int priority, 333 const char *wmesg, uint64_t nsecs) 334 { 335 uint64_t to_ticks; 336 337 if (nsecs == INFSLP) 338 return rwsleep(ident, rwl, priority, wmesg, 0); 339 #ifdef DIAGNOSTIC 340 if (nsecs == 0) { 341 log(LOG_WARNING, 342 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 343 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 344 wmesg); 345 } 346 #endif 347 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 348 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 349 if (to_ticks > INT_MAX) 350 to_ticks = INT_MAX; 351 return rwsleep(ident, rwl, priority, wmesg, (int)to_ticks); 352 } 353 354 void 355 sleep_setup(struct sleep_state *sls, const volatile void *ident, int prio, 356 const char *wmesg) 357 { 358 struct proc *p = curproc; 359 360 #ifdef DIAGNOSTIC 361 if (p->p_flag & P_CANTSLEEP) 362 panic("sleep: %s failed insomnia", p->p_p->ps_comm); 363 if (ident == NULL) 364 panic("tsleep: no ident"); 365 if (p->p_stat != SONPROC) 366 panic("tsleep: not SONPROC"); 367 #endif 368 369 sls->sls_catch = prio & PCATCH; 370 sls->sls_do_sleep = 1; 371 sls->sls_locked = 0; 372 sls->sls_sigerr = 0; 373 sls->sls_timeout = 0; 374 375 /* 376 * The kernel has to be locked for signal processing. 377 * This is done here and not in sleep_setup_signal() because 378 * KERNEL_LOCK() has to be taken before SCHED_LOCK(). 379 */ 380 if (sls->sls_catch != 0) { 381 KERNEL_LOCK(); 382 sls->sls_locked = 1; 383 } 384 385 SCHED_LOCK(sls->sls_s); 386 387 TRACEPOINT(sched, sleep, NULL); 388 389 p->p_wchan = ident; 390 p->p_wmesg = wmesg; 391 p->p_slptime = 0; 392 p->p_slppri = prio & PRIMASK; 393 TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], p, p_runq); 394 } 395 396 int 397 sleep_finish_all(struct sleep_state *sls, int do_sleep) 398 { 399 int error, error1; 400 401 sleep_finish(sls, do_sleep); 402 error1 = sleep_finish_timeout(sls); 403 error = sleep_finish_signal(sls); 404 405 /* Signal errors are higher priority than timeouts. */ 406 if (error == 0 && error1 != 0) 407 error = error1; 408 409 return error; 410 } 411 412 void 413 sleep_finish(struct sleep_state *sls, int do_sleep) 414 { 415 struct proc *p = curproc; 416 417 if (sls->sls_do_sleep && do_sleep) { 418 p->p_stat = SSLEEP; 419 p->p_ru.ru_nvcsw++; 420 SCHED_ASSERT_LOCKED(); 421 mi_switch(); 422 } else if (!do_sleep) { 423 unsleep(p); 424 } 425 426 #ifdef DIAGNOSTIC 427 if (p->p_stat != SONPROC) 428 panic("sleep_finish !SONPROC"); 429 #endif 430 431 p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri; 432 SCHED_UNLOCK(sls->sls_s); 433 434 /* 435 * Even though this belongs to the signal handling part of sleep, 436 * we need to clear it before the ktrace. 437 */ 438 atomic_clearbits_int(&p->p_flag, P_SINTR); 439 } 440 441 void 442 sleep_setup_timeout(struct sleep_state *sls, int timo) 443 { 444 struct proc *p = curproc; 445 446 KASSERT((p->p_flag & P_TIMEOUT) == 0); 447 448 if (timo) { 449 sls->sls_timeout = 1; 450 timeout_add(&p->p_sleep_to, timo); 451 } 452 } 453 454 int 455 sleep_finish_timeout(struct sleep_state *sls) 456 { 457 struct proc *p = curproc; 458 459 if (sls->sls_timeout) { 460 if (p->p_flag & P_TIMEOUT) { 461 atomic_clearbits_int(&p->p_flag, P_TIMEOUT); 462 return (EWOULDBLOCK); 463 } else { 464 /* This must not sleep. */ 465 timeout_del_barrier(&p->p_sleep_to); 466 KASSERT((p->p_flag & P_TIMEOUT) == 0); 467 } 468 } 469 470 return (0); 471 } 472 473 int 474 sleep_signal_check(struct proc *p) 475 { 476 int err, sig; 477 478 if ((err = single_thread_check(p, 1)) != 0) 479 return err; 480 if ((sig = CURSIG(p)) != 0) { 481 if (p->p_p->ps_sigacts->ps_sigintr & sigmask(sig)) 482 return EINTR; 483 else 484 return ERESTART; 485 } 486 return 0; 487 } 488 489 void 490 sleep_setup_signal(struct sleep_state *sls) 491 { 492 struct proc *p = curproc; 493 494 if (sls->sls_catch == 0) 495 return; 496 497 /* sleep_setup() has locked the kernel. */ 498 KERNEL_ASSERT_LOCKED(); 499 500 /* 501 * We put ourselves on the sleep queue and start our timeout before 502 * calling single_thread_check or CURSIG, as we could stop there, and 503 * a wakeup or a SIGCONT (or both) could occur while we were stopped. 504 * A SIGCONT would cause us to be marked as SSLEEP without resuming us, 505 * thus we must be ready for sleep when CURSIG is called. If the 506 * wakeup happens while we're stopped, p->p_wchan will be 0 upon 507 * return from single_thread_check or CURSIG. In that case we should 508 * not go to sleep. If single_thread_check returns an error we need 509 * to unwind immediately. That's achieved by saving the return value 510 * in sls->sl_unwind and checking it later in sleep_finish_signal. 511 */ 512 atomic_setbits_int(&p->p_flag, P_SINTR); 513 if ((sls->sls_sigerr = sleep_signal_check(p)) != 0) { 514 unsleep(p); 515 p->p_stat = SONPROC; 516 sls->sls_do_sleep = 0; 517 } else if (p->p_wchan == 0) { 518 sls->sls_catch = 0; 519 sls->sls_do_sleep = 0; 520 } 521 } 522 523 int 524 sleep_finish_signal(struct sleep_state *sls) 525 { 526 struct proc *p = curproc; 527 int error = 0; 528 529 if (sls->sls_catch != 0) { 530 KERNEL_ASSERT_LOCKED(); 531 532 if (sls->sls_sigerr != 0) 533 error = sls->sls_sigerr; 534 else 535 error = sleep_signal_check(p); 536 } 537 538 if (sls->sls_locked) 539 KERNEL_UNLOCK(); 540 541 return (error); 542 } 543 544 int 545 wakeup_proc(struct proc *p, const volatile void *chan) 546 { 547 int s, awakened = 0; 548 549 SCHED_LOCK(s); 550 if (p->p_wchan != NULL && 551 ((chan == NULL) || (p->p_wchan == chan))) { 552 awakened = 1; 553 if (p->p_stat == SSLEEP) 554 setrunnable(p); 555 else 556 unsleep(p); 557 } 558 SCHED_UNLOCK(s); 559 560 return awakened; 561 } 562 563 /* 564 * Implement timeout for tsleep. 565 * If process hasn't been awakened (wchan non-zero), 566 * set timeout flag and undo the sleep. If proc 567 * is stopped, just unsleep so it will remain stopped. 568 */ 569 void 570 endtsleep(void *arg) 571 { 572 struct proc *p = arg; 573 int s; 574 575 SCHED_LOCK(s); 576 if (wakeup_proc(p, NULL)) 577 atomic_setbits_int(&p->p_flag, P_TIMEOUT); 578 SCHED_UNLOCK(s); 579 } 580 581 /* 582 * Remove a process from its wait queue 583 */ 584 void 585 unsleep(struct proc *p) 586 { 587 SCHED_ASSERT_LOCKED(); 588 589 if (p->p_wchan != NULL) { 590 TAILQ_REMOVE(&slpque[LOOKUP(p->p_wchan)], p, p_runq); 591 p->p_wchan = NULL; 592 TRACEPOINT(sched, wakeup, p->p_tid, p->p_p->ps_pid); 593 } 594 } 595 596 /* 597 * Make a number of processes sleeping on the specified identifier runnable. 598 */ 599 void 600 wakeup_n(const volatile void *ident, int n) 601 { 602 struct slpque *qp; 603 struct proc *p; 604 struct proc *pnext; 605 int s; 606 607 SCHED_LOCK(s); 608 qp = &slpque[LOOKUP(ident)]; 609 for (p = TAILQ_FIRST(qp); p != NULL && n != 0; p = pnext) { 610 pnext = TAILQ_NEXT(p, p_runq); 611 #ifdef DIAGNOSTIC 612 /* 613 * If the rwlock passed to rwsleep() is contended, the 614 * CPU will end up calling wakeup() between sleep_setup() 615 * and sleep_finish(). 616 */ 617 if (p == curproc) { 618 KASSERT(p->p_stat == SONPROC); 619 continue; 620 } 621 if (p->p_stat != SSLEEP && p->p_stat != SSTOP) 622 panic("wakeup: p_stat is %d", (int)p->p_stat); 623 #endif 624 if (wakeup_proc(p, ident)) 625 --n; 626 } 627 SCHED_UNLOCK(s); 628 } 629 630 /* 631 * Make all processes sleeping on the specified identifier runnable. 632 */ 633 void 634 wakeup(const volatile void *chan) 635 { 636 wakeup_n(chan, -1); 637 } 638 639 int 640 sys_sched_yield(struct proc *p, void *v, register_t *retval) 641 { 642 struct proc *q; 643 uint8_t newprio; 644 int s; 645 646 SCHED_LOCK(s); 647 /* 648 * If one of the threads of a multi-threaded process called 649 * sched_yield(2), drop its priority to ensure its siblings 650 * can make some progress. 651 */ 652 newprio = p->p_usrpri; 653 SMR_TAILQ_FOREACH_LOCKED(q, &p->p_p->ps_threads, p_thr_link) 654 newprio = max(newprio, q->p_runpri); 655 setrunqueue(p->p_cpu, p, newprio); 656 p->p_ru.ru_nvcsw++; 657 mi_switch(); 658 SCHED_UNLOCK(s); 659 660 return (0); 661 } 662 663 int 664 thrsleep_unlock(void *lock) 665 { 666 static _atomic_lock_t unlocked = _ATOMIC_LOCK_UNLOCKED; 667 _atomic_lock_t *atomiclock = lock; 668 669 if (!lock) 670 return 0; 671 672 return copyout(&unlocked, atomiclock, sizeof(unlocked)); 673 } 674 675 struct tslpentry { 676 TAILQ_ENTRY(tslpentry) tslp_link; 677 long tslp_ident; 678 }; 679 680 /* thrsleep queue shared between processes */ 681 static struct tslpqueue thrsleep_queue = TAILQ_HEAD_INITIALIZER(thrsleep_queue); 682 static struct rwlock thrsleep_lock = RWLOCK_INITIALIZER("thrsleeplk"); 683 684 int 685 thrsleep(struct proc *p, struct sys___thrsleep_args *v) 686 { 687 struct sys___thrsleep_args /* { 688 syscallarg(const volatile void *) ident; 689 syscallarg(clockid_t) clock_id; 690 syscallarg(const struct timespec *) tp; 691 syscallarg(void *) lock; 692 syscallarg(const int *) abort; 693 } */ *uap = v; 694 long ident = (long)SCARG(uap, ident); 695 struct tslpentry entry; 696 struct tslpqueue *queue; 697 struct rwlock *qlock; 698 struct timespec *tsp = (struct timespec *)SCARG(uap, tp); 699 void *lock = SCARG(uap, lock); 700 uint64_t nsecs = INFSLP; 701 int abort = 0, error; 702 clockid_t clock_id = SCARG(uap, clock_id); 703 704 if (ident == 0) 705 return (EINVAL); 706 if (tsp != NULL) { 707 struct timespec now; 708 709 if ((error = clock_gettime(p, clock_id, &now))) 710 return (error); 711 #ifdef KTRACE 712 if (KTRPOINT(p, KTR_STRUCT)) 713 ktrabstimespec(p, tsp); 714 #endif 715 716 if (timespeccmp(tsp, &now, <=)) { 717 /* already passed: still do the unlock */ 718 if ((error = thrsleep_unlock(lock))) 719 return (error); 720 return (EWOULDBLOCK); 721 } 722 723 timespecsub(tsp, &now, tsp); 724 nsecs = MIN(TIMESPEC_TO_NSEC(tsp), MAXTSLP); 725 } 726 727 if (ident == -1) { 728 queue = &thrsleep_queue; 729 qlock = &thrsleep_lock; 730 } else { 731 queue = &p->p_p->ps_tslpqueue; 732 qlock = &p->p_p->ps_lock; 733 } 734 735 /* Interlock with wakeup. */ 736 entry.tslp_ident = ident; 737 rw_enter_write(qlock); 738 TAILQ_INSERT_TAIL(queue, &entry, tslp_link); 739 rw_exit_write(qlock); 740 741 error = thrsleep_unlock(lock); 742 743 if (error == 0 && SCARG(uap, abort) != NULL) 744 error = copyin(SCARG(uap, abort), &abort, sizeof(abort)); 745 746 rw_enter_write(qlock); 747 if (error != 0) 748 goto out; 749 if (abort != 0) { 750 error = EINTR; 751 goto out; 752 } 753 if (entry.tslp_ident != 0) { 754 error = rwsleep_nsec(&entry, qlock, PWAIT|PCATCH, "thrsleep", 755 nsecs); 756 } 757 758 out: 759 if (entry.tslp_ident != 0) 760 TAILQ_REMOVE(queue, &entry, tslp_link); 761 rw_exit_write(qlock); 762 763 if (error == ERESTART) 764 error = ECANCELED; 765 766 return (error); 767 768 } 769 770 int 771 sys___thrsleep(struct proc *p, void *v, register_t *retval) 772 { 773 struct sys___thrsleep_args /* { 774 syscallarg(const volatile void *) ident; 775 syscallarg(clockid_t) clock_id; 776 syscallarg(struct timespec *) tp; 777 syscallarg(void *) lock; 778 syscallarg(const int *) abort; 779 } */ *uap = v; 780 struct timespec ts; 781 int error; 782 783 if (SCARG(uap, tp) != NULL) { 784 if ((error = copyin(SCARG(uap, tp), &ts, sizeof(ts)))) { 785 *retval = error; 786 return 0; 787 } 788 if (!timespecisvalid(&ts)) { 789 *retval = EINVAL; 790 return 0; 791 } 792 SCARG(uap, tp) = &ts; 793 } 794 795 *retval = thrsleep(p, uap); 796 return 0; 797 } 798 799 int 800 sys___thrwakeup(struct proc *p, void *v, register_t *retval) 801 { 802 struct sys___thrwakeup_args /* { 803 syscallarg(const volatile void *) ident; 804 syscallarg(int) n; 805 } */ *uap = v; 806 struct tslpentry *entry, *tmp; 807 struct tslpqueue *queue; 808 struct rwlock *qlock; 809 long ident = (long)SCARG(uap, ident); 810 int n = SCARG(uap, n); 811 int found = 0; 812 813 if (ident == 0) 814 *retval = EINVAL; 815 else { 816 if (ident == -1) { 817 queue = &thrsleep_queue; 818 qlock = &thrsleep_lock; 819 /* 820 * Wake up all waiters with ident -1. This is needed 821 * because ident -1 can be shared by multiple userspace 822 * lock state machines concurrently. The implementation 823 * has no way to direct the wakeup to a particular 824 * state machine. 825 */ 826 n = 0; 827 } else { 828 queue = &p->p_p->ps_tslpqueue; 829 qlock = &p->p_p->ps_lock; 830 } 831 832 rw_enter_write(qlock); 833 TAILQ_FOREACH_SAFE(entry, queue, tslp_link, tmp) { 834 if (entry->tslp_ident == ident) { 835 TAILQ_REMOVE(queue, entry, tslp_link); 836 entry->tslp_ident = 0; 837 wakeup_one(entry); 838 if (++found == n) 839 break; 840 } 841 } 842 rw_exit_write(qlock); 843 844 if (ident == -1) 845 *retval = 0; 846 else 847 *retval = found ? 0 : ESRCH; 848 } 849 850 return (0); 851 } 852 853 void 854 refcnt_init(struct refcnt *r) 855 { 856 r->refs = 1; 857 } 858 859 void 860 refcnt_take(struct refcnt *r) 861 { 862 #ifdef DIAGNOSTIC 863 u_int refcnt; 864 865 refcnt = atomic_inc_int_nv(&r->refs); 866 KASSERT(refcnt != 0); 867 #else 868 atomic_inc_int(&r->refs); 869 #endif 870 } 871 872 int 873 refcnt_rele(struct refcnt *r) 874 { 875 u_int refcnt; 876 877 refcnt = atomic_dec_int_nv(&r->refs); 878 KASSERT(refcnt != ~0); 879 880 return (refcnt == 0); 881 } 882 883 void 884 refcnt_rele_wake(struct refcnt *r) 885 { 886 if (refcnt_rele(r)) 887 wakeup_one(r); 888 } 889 890 void 891 refcnt_finalize(struct refcnt *r, const char *wmesg) 892 { 893 struct sleep_state sls; 894 u_int refcnt; 895 896 refcnt = atomic_dec_int_nv(&r->refs); 897 while (refcnt) { 898 sleep_setup(&sls, r, PWAIT, wmesg); 899 refcnt = r->refs; 900 sleep_finish(&sls, refcnt); 901 } 902 } 903 904 void 905 cond_init(struct cond *c) 906 { 907 c->c_wait = 1; 908 } 909 910 void 911 cond_signal(struct cond *c) 912 { 913 c->c_wait = 0; 914 915 wakeup_one(c); 916 } 917 918 void 919 cond_wait(struct cond *c, const char *wmesg) 920 { 921 struct sleep_state sls; 922 int wait; 923 924 wait = c->c_wait; 925 while (wait) { 926 sleep_setup(&sls, c, PWAIT, wmesg); 927 wait = c->c_wait; 928 sleep_finish(&sls, wait); 929 } 930 } 931