1 /* $OpenBSD: kern_synch.c,v 1.173 2020/12/24 01:16:14 cheloha Exp $ */ 2 /* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */ 3 4 /* 5 * Copyright (c) 1982, 1986, 1990, 1991, 1993 6 * The Regents of the University of California. All rights reserved. 7 * (c) UNIX System Laboratories, Inc. 8 * All or some portions of this file are derived from material licensed 9 * to the University of California by American Telephone and Telegraph 10 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 11 * the permission of UNIX System Laboratories, Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * @(#)kern_synch.c 8.6 (Berkeley) 1/21/94 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/proc.h> 43 #include <sys/kernel.h> 44 #include <sys/signalvar.h> 45 #include <sys/resourcevar.h> 46 #include <sys/sched.h> 47 #include <sys/timeout.h> 48 #include <sys/mount.h> 49 #include <sys/syscallargs.h> 50 #include <sys/pool.h> 51 #include <sys/refcnt.h> 52 #include <sys/atomic.h> 53 #include <sys/smr.h> 54 #include <sys/witness.h> 55 #include <sys/tracepoint.h> 56 57 #include <ddb/db_output.h> 58 59 #include <machine/spinlock.h> 60 61 #ifdef DIAGNOSTIC 62 #include <sys/syslog.h> 63 #endif 64 65 #ifdef KTRACE 66 #include <sys/ktrace.h> 67 #endif 68 69 int thrsleep(struct proc *, struct sys___thrsleep_args *); 70 int thrsleep_unlock(void *); 71 72 /* 73 * We're only looking at 7 bits of the address; everything is 74 * aligned to 4, lots of things are aligned to greater powers 75 * of 2. Shift right by 8, i.e. drop the bottom 256 worth. 76 */ 77 #define TABLESIZE 128 78 #define LOOKUP(x) (((long)(x) >> 8) & (TABLESIZE - 1)) 79 TAILQ_HEAD(slpque,proc) slpque[TABLESIZE]; 80 81 void 82 sleep_queue_init(void) 83 { 84 int i; 85 86 for (i = 0; i < TABLESIZE; i++) 87 TAILQ_INIT(&slpque[i]); 88 } 89 90 /* 91 * Global sleep channel for threads that do not want to 92 * receive wakeup(9) broadcasts. 93 */ 94 int nowake; 95 96 /* 97 * During autoconfiguration or after a panic, a sleep will simply 98 * lower the priority briefly to allow interrupts, then return. 99 * The priority to be used (safepri) is machine-dependent, thus this 100 * value is initialized and maintained in the machine-dependent layers. 101 * This priority will typically be 0, or the lowest priority 102 * that is safe for use on the interrupt stack; it can be made 103 * higher to block network software interrupts after panics. 104 */ 105 extern int safepri; 106 107 /* 108 * General sleep call. Suspends the current process until a wakeup is 109 * performed on the specified identifier. The process will then be made 110 * runnable with the specified priority. Sleeps at most timo/hz seconds 111 * (0 means no timeout). If pri includes PCATCH flag, signals are checked 112 * before and after sleeping, else signals are not checked. Returns 0 if 113 * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 114 * signal needs to be delivered, ERESTART is returned if the current system 115 * call should be restarted if possible, and EINTR is returned if the system 116 * call should be interrupted by the signal (return EINTR). 117 */ 118 int 119 tsleep(const volatile void *ident, int priority, const char *wmesg, int timo) 120 { 121 struct sleep_state sls; 122 #ifdef MULTIPROCESSOR 123 int hold_count; 124 #endif 125 126 KASSERT((priority & ~(PRIMASK | PCATCH)) == 0); 127 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 128 129 #ifdef MULTIPROCESSOR 130 KASSERT(timo || _kernel_lock_held()); 131 #endif 132 133 #ifdef DDB 134 if (cold == 2) 135 db_stack_dump(); 136 #endif 137 if (cold || panicstr) { 138 int s; 139 /* 140 * After a panic, or during autoconfiguration, 141 * just give interrupts a chance, then just return; 142 * don't run any other procs or panic below, 143 * in case this is the idle process and already asleep. 144 */ 145 s = splhigh(); 146 splx(safepri); 147 #ifdef MULTIPROCESSOR 148 if (_kernel_lock_held()) { 149 hold_count = __mp_release_all(&kernel_lock); 150 __mp_acquire_count(&kernel_lock, hold_count); 151 } 152 #endif 153 splx(s); 154 return (0); 155 } 156 157 sleep_setup(&sls, ident, priority, wmesg); 158 sleep_setup_timeout(&sls, timo); 159 sleep_setup_signal(&sls); 160 161 return sleep_finish_all(&sls, 1); 162 } 163 164 int 165 tsleep_nsec(const volatile void *ident, int priority, const char *wmesg, 166 uint64_t nsecs) 167 { 168 uint64_t to_ticks; 169 170 if (nsecs == INFSLP) 171 return tsleep(ident, priority, wmesg, 0); 172 #ifdef DIAGNOSTIC 173 if (nsecs == 0) { 174 log(LOG_WARNING, 175 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 176 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 177 wmesg); 178 } 179 #endif 180 /* 181 * We want to sleep at least nsecs nanoseconds worth of ticks. 182 * 183 * - Clamp nsecs to prevent arithmetic overflow. 184 * 185 * - Round nsecs up to account for any nanoseconds that do not 186 * divide evenly into tick_nsec, otherwise we'll lose them to 187 * integer division in the next step. We add (tick_nsec - 1) 188 * to keep from introducing a spurious tick if there are no 189 * such nanoseconds, i.e. nsecs % tick_nsec == 0. 190 * 191 * - Divide the rounded value to a count of ticks. We divide 192 * by (tick_nsec + 1) to discard the extra tick introduced if, 193 * before rounding, nsecs % tick_nsec == 1. 194 * 195 * - Finally, add a tick to the result. We need to wait out 196 * the current tick before we can begin counting our interval, 197 * as we do not know how much time has elapsed since the 198 * current tick began. 199 */ 200 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 201 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 202 if (to_ticks > INT_MAX) 203 to_ticks = INT_MAX; 204 return tsleep(ident, priority, wmesg, (int)to_ticks); 205 } 206 207 /* 208 * Same as tsleep, but if we have a mutex provided, then once we've 209 * entered the sleep queue we drop the mutex. After sleeping we re-lock. 210 */ 211 int 212 msleep(const volatile void *ident, struct mutex *mtx, int priority, 213 const char *wmesg, int timo) 214 { 215 struct sleep_state sls; 216 int error, spl; 217 #ifdef MULTIPROCESSOR 218 int hold_count; 219 #endif 220 221 KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); 222 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 223 KASSERT(mtx != NULL); 224 225 if (priority & PCATCH) 226 KERNEL_ASSERT_LOCKED(); 227 228 if (cold || panicstr) { 229 /* 230 * After a panic, or during autoconfiguration, 231 * just give interrupts a chance, then just return; 232 * don't run any other procs or panic below, 233 * in case this is the idle process and already asleep. 234 */ 235 spl = MUTEX_OLDIPL(mtx); 236 MUTEX_OLDIPL(mtx) = safepri; 237 mtx_leave(mtx); 238 #ifdef MULTIPROCESSOR 239 if (_kernel_lock_held()) { 240 hold_count = __mp_release_all(&kernel_lock); 241 __mp_acquire_count(&kernel_lock, hold_count); 242 } 243 #endif 244 if ((priority & PNORELOCK) == 0) { 245 mtx_enter(mtx); 246 MUTEX_OLDIPL(mtx) = spl; 247 } else 248 splx(spl); 249 return (0); 250 } 251 252 sleep_setup(&sls, ident, priority, wmesg); 253 sleep_setup_timeout(&sls, timo); 254 255 /* XXX - We need to make sure that the mutex doesn't 256 * unblock splsched. This can be made a bit more 257 * correct when the sched_lock is a mutex. 258 */ 259 spl = MUTEX_OLDIPL(mtx); 260 MUTEX_OLDIPL(mtx) = splsched(); 261 mtx_leave(mtx); 262 /* signal may stop the process, release mutex before that */ 263 sleep_setup_signal(&sls); 264 265 error = sleep_finish_all(&sls, 1); 266 267 if ((priority & PNORELOCK) == 0) { 268 mtx_enter(mtx); 269 MUTEX_OLDIPL(mtx) = spl; /* put the ipl back */ 270 } else 271 splx(spl); 272 273 return error; 274 } 275 276 int 277 msleep_nsec(const volatile void *ident, struct mutex *mtx, int priority, 278 const char *wmesg, uint64_t nsecs) 279 { 280 uint64_t to_ticks; 281 282 if (nsecs == INFSLP) 283 return msleep(ident, mtx, priority, wmesg, 0); 284 #ifdef DIAGNOSTIC 285 if (nsecs == 0) { 286 log(LOG_WARNING, 287 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 288 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 289 wmesg); 290 } 291 #endif 292 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 293 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 294 if (to_ticks > INT_MAX) 295 to_ticks = INT_MAX; 296 return msleep(ident, mtx, priority, wmesg, (int)to_ticks); 297 } 298 299 /* 300 * Same as tsleep, but if we have a rwlock provided, then once we've 301 * entered the sleep queue we drop the it. After sleeping we re-lock. 302 */ 303 int 304 rwsleep(const volatile void *ident, struct rwlock *rwl, int priority, 305 const char *wmesg, int timo) 306 { 307 struct sleep_state sls; 308 int error, status; 309 310 KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0); 311 KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0); 312 rw_assert_anylock(rwl); 313 status = rw_status(rwl); 314 315 sleep_setup(&sls, ident, priority, wmesg); 316 sleep_setup_timeout(&sls, timo); 317 318 rw_exit(rwl); 319 /* signal may stop the process, release rwlock before that */ 320 sleep_setup_signal(&sls); 321 322 error = sleep_finish_all(&sls, 1); 323 324 if ((priority & PNORELOCK) == 0) 325 rw_enter(rwl, status); 326 327 return error; 328 } 329 330 int 331 rwsleep_nsec(const volatile void *ident, struct rwlock *rwl, int priority, 332 const char *wmesg, uint64_t nsecs) 333 { 334 uint64_t to_ticks; 335 336 if (nsecs == INFSLP) 337 return rwsleep(ident, rwl, priority, wmesg, 0); 338 #ifdef DIAGNOSTIC 339 if (nsecs == 0) { 340 log(LOG_WARNING, 341 "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n", 342 __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid, 343 wmesg); 344 } 345 #endif 346 nsecs = MIN(nsecs, UINT64_MAX - tick_nsec); 347 to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1; 348 if (to_ticks > INT_MAX) 349 to_ticks = INT_MAX; 350 return rwsleep(ident, rwl, priority, wmesg, (int)to_ticks); 351 } 352 353 void 354 sleep_setup(struct sleep_state *sls, const volatile void *ident, int prio, 355 const char *wmesg) 356 { 357 struct proc *p = curproc; 358 359 #ifdef DIAGNOSTIC 360 if (p->p_flag & P_CANTSLEEP) 361 panic("sleep: %s failed insomnia", p->p_p->ps_comm); 362 if (ident == NULL) 363 panic("tsleep: no ident"); 364 if (p->p_stat != SONPROC) 365 panic("tsleep: not SONPROC"); 366 #endif 367 368 sls->sls_catch = prio & PCATCH; 369 sls->sls_do_sleep = 1; 370 sls->sls_locked = 0; 371 sls->sls_sig = 0; 372 sls->sls_unwind = 0; 373 sls->sls_timeout = 0; 374 375 /* 376 * The kernel has to be locked for signal processing. 377 * This is done here and not in sleep_setup_signal() because 378 * KERNEL_LOCK() has to be taken before SCHED_LOCK(). 379 */ 380 if (sls->sls_catch != 0) { 381 KERNEL_LOCK(); 382 sls->sls_locked = 1; 383 } 384 385 SCHED_LOCK(sls->sls_s); 386 387 TRACEPOINT(sched, sleep, NULL); 388 389 p->p_wchan = ident; 390 p->p_wmesg = wmesg; 391 p->p_slptime = 0; 392 p->p_slppri = prio & PRIMASK; 393 TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], p, p_runq); 394 } 395 396 int 397 sleep_finish_all(struct sleep_state *sls, int do_sleep) 398 { 399 int error, error1; 400 401 sleep_finish(sls, do_sleep); 402 error1 = sleep_finish_timeout(sls); 403 error = sleep_finish_signal(sls); 404 405 /* Signal errors are higher priority than timeouts. */ 406 if (error == 0 && error1 != 0) 407 error = error1; 408 409 return error; 410 } 411 412 void 413 sleep_finish(struct sleep_state *sls, int do_sleep) 414 { 415 struct proc *p = curproc; 416 417 if (sls->sls_do_sleep && do_sleep) { 418 p->p_stat = SSLEEP; 419 p->p_ru.ru_nvcsw++; 420 SCHED_ASSERT_LOCKED(); 421 mi_switch(); 422 } else if (!do_sleep) { 423 unsleep(p); 424 } 425 426 #ifdef DIAGNOSTIC 427 if (p->p_stat != SONPROC) 428 panic("sleep_finish !SONPROC"); 429 #endif 430 431 p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri; 432 SCHED_UNLOCK(sls->sls_s); 433 434 /* 435 * Even though this belongs to the signal handling part of sleep, 436 * we need to clear it before the ktrace. 437 */ 438 atomic_clearbits_int(&p->p_flag, P_SINTR); 439 } 440 441 void 442 sleep_setup_timeout(struct sleep_state *sls, int timo) 443 { 444 struct proc *p = curproc; 445 446 KASSERT((p->p_flag & P_TIMEOUT) == 0); 447 448 if (timo) { 449 sls->sls_timeout = 1; 450 timeout_add(&p->p_sleep_to, timo); 451 } 452 } 453 454 int 455 sleep_finish_timeout(struct sleep_state *sls) 456 { 457 struct proc *p = curproc; 458 459 if (sls->sls_timeout) { 460 if (p->p_flag & P_TIMEOUT) { 461 atomic_clearbits_int(&p->p_flag, P_TIMEOUT); 462 return (EWOULDBLOCK); 463 } else { 464 /* This must not sleep. */ 465 timeout_del_barrier(&p->p_sleep_to); 466 KASSERT((p->p_flag & P_TIMEOUT) == 0); 467 } 468 } 469 470 return (0); 471 } 472 473 void 474 sleep_setup_signal(struct sleep_state *sls) 475 { 476 struct proc *p = curproc; 477 478 if (sls->sls_catch == 0) 479 return; 480 481 /* sleep_setup() has locked the kernel. */ 482 KERNEL_ASSERT_LOCKED(); 483 484 /* 485 * We put ourselves on the sleep queue and start our timeout before 486 * calling single_thread_check or CURSIG, as we could stop there, and 487 * a wakeup or a SIGCONT (or both) could occur while we were stopped. 488 * A SIGCONT would cause us to be marked as SSLEEP without resuming us, 489 * thus we must be ready for sleep when CURSIG is called. If the 490 * wakeup happens while we're stopped, p->p_wchan will be 0 upon 491 * return from single_thread_check or CURSIG. In that case we should 492 * not go to sleep. If single_thread_check returns an error we need 493 * to unwind immediately. That's achieved by saving the return value 494 * in sls->sl_unwind and checking it later in sleep_finish_signal. 495 */ 496 atomic_setbits_int(&p->p_flag, P_SINTR); 497 if ((sls->sls_unwind = single_thread_check(p, 1)) != 0 || 498 (sls->sls_sig = CURSIG(p)) != 0) { 499 unsleep(p); 500 p->p_stat = SONPROC; 501 sls->sls_do_sleep = 0; 502 } else if (p->p_wchan == 0) { 503 sls->sls_catch = 0; 504 sls->sls_do_sleep = 0; 505 } 506 } 507 508 int 509 sleep_finish_signal(struct sleep_state *sls) 510 { 511 struct proc *p = curproc; 512 int error = 0; 513 514 if (sls->sls_catch != 0) { 515 KERNEL_ASSERT_LOCKED(); 516 517 if (sls->sls_unwind != 0 || 518 (sls->sls_unwind = single_thread_check(p, 1)) != 0) 519 error = sls->sls_unwind; 520 else if (sls->sls_sig != 0 || 521 (sls->sls_sig = CURSIG(p)) != 0) { 522 if (p->p_p->ps_sigacts->ps_sigintr & 523 sigmask(sls->sls_sig)) 524 error = EINTR; 525 else 526 error = ERESTART; 527 } 528 } 529 530 if (sls->sls_locked) 531 KERNEL_UNLOCK(); 532 533 return (error); 534 } 535 536 int 537 wakeup_proc(struct proc *p, const volatile void *chan) 538 { 539 int s, awakened = 0; 540 541 SCHED_LOCK(s); 542 if (p->p_wchan != NULL && 543 ((chan == NULL) || (p->p_wchan == chan))) { 544 awakened = 1; 545 if (p->p_stat == SSLEEP) 546 setrunnable(p); 547 else 548 unsleep(p); 549 } 550 SCHED_UNLOCK(s); 551 552 return awakened; 553 } 554 555 /* 556 * Implement timeout for tsleep. 557 * If process hasn't been awakened (wchan non-zero), 558 * set timeout flag and undo the sleep. If proc 559 * is stopped, just unsleep so it will remain stopped. 560 */ 561 void 562 endtsleep(void *arg) 563 { 564 struct proc *p = arg; 565 int s; 566 567 SCHED_LOCK(s); 568 if (wakeup_proc(p, NULL)) 569 atomic_setbits_int(&p->p_flag, P_TIMEOUT); 570 SCHED_UNLOCK(s); 571 } 572 573 /* 574 * Remove a process from its wait queue 575 */ 576 void 577 unsleep(struct proc *p) 578 { 579 SCHED_ASSERT_LOCKED(); 580 581 if (p->p_wchan != NULL) { 582 TAILQ_REMOVE(&slpque[LOOKUP(p->p_wchan)], p, p_runq); 583 p->p_wchan = NULL; 584 TRACEPOINT(sched, wakeup, p->p_tid, p->p_p->ps_pid); 585 } 586 } 587 588 /* 589 * Make a number of processes sleeping on the specified identifier runnable. 590 */ 591 void 592 wakeup_n(const volatile void *ident, int n) 593 { 594 struct slpque *qp; 595 struct proc *p; 596 struct proc *pnext; 597 int s; 598 599 SCHED_LOCK(s); 600 qp = &slpque[LOOKUP(ident)]; 601 for (p = TAILQ_FIRST(qp); p != NULL && n != 0; p = pnext) { 602 pnext = TAILQ_NEXT(p, p_runq); 603 #ifdef DIAGNOSTIC 604 /* 605 * If the rwlock passed to rwsleep() is contended, the 606 * CPU will end up calling wakeup() between sleep_setup() 607 * and sleep_finish(). 608 */ 609 if (p == curproc) { 610 KASSERT(p->p_stat == SONPROC); 611 continue; 612 } 613 if (p->p_stat != SSLEEP && p->p_stat != SSTOP) 614 panic("wakeup: p_stat is %d", (int)p->p_stat); 615 #endif 616 if (wakeup_proc(p, ident)) 617 --n; 618 } 619 SCHED_UNLOCK(s); 620 } 621 622 /* 623 * Make all processes sleeping on the specified identifier runnable. 624 */ 625 void 626 wakeup(const volatile void *chan) 627 { 628 wakeup_n(chan, -1); 629 } 630 631 int 632 sys_sched_yield(struct proc *p, void *v, register_t *retval) 633 { 634 struct proc *q; 635 uint8_t newprio; 636 int s; 637 638 SCHED_LOCK(s); 639 /* 640 * If one of the threads of a multi-threaded process called 641 * sched_yield(2), drop its priority to ensure its siblings 642 * can make some progress. 643 */ 644 newprio = p->p_usrpri; 645 SMR_TAILQ_FOREACH_LOCKED(q, &p->p_p->ps_threads, p_thr_link) 646 newprio = max(newprio, q->p_runpri); 647 setrunqueue(p->p_cpu, p, newprio); 648 p->p_ru.ru_nvcsw++; 649 mi_switch(); 650 SCHED_UNLOCK(s); 651 652 return (0); 653 } 654 655 int 656 thrsleep_unlock(void *lock) 657 { 658 static _atomic_lock_t unlocked = _ATOMIC_LOCK_UNLOCKED; 659 _atomic_lock_t *atomiclock = lock; 660 661 if (!lock) 662 return 0; 663 664 return copyout(&unlocked, atomiclock, sizeof(unlocked)); 665 } 666 667 struct tslpentry { 668 TAILQ_ENTRY(tslpentry) tslp_link; 669 long tslp_ident; 670 }; 671 672 /* thrsleep queue shared between processes */ 673 static struct tslpqueue thrsleep_queue = TAILQ_HEAD_INITIALIZER(thrsleep_queue); 674 static struct rwlock thrsleep_lock = RWLOCK_INITIALIZER("thrsleeplk"); 675 676 int 677 thrsleep(struct proc *p, struct sys___thrsleep_args *v) 678 { 679 struct sys___thrsleep_args /* { 680 syscallarg(const volatile void *) ident; 681 syscallarg(clockid_t) clock_id; 682 syscallarg(const struct timespec *) tp; 683 syscallarg(void *) lock; 684 syscallarg(const int *) abort; 685 } */ *uap = v; 686 long ident = (long)SCARG(uap, ident); 687 struct tslpentry entry; 688 struct tslpqueue *queue; 689 struct rwlock *qlock; 690 struct timespec *tsp = (struct timespec *)SCARG(uap, tp); 691 void *lock = SCARG(uap, lock); 692 uint64_t nsecs = INFSLP; 693 int abort = 0, error; 694 clockid_t clock_id = SCARG(uap, clock_id); 695 696 if (ident == 0) 697 return (EINVAL); 698 if (tsp != NULL) { 699 struct timespec now; 700 701 if ((error = clock_gettime(p, clock_id, &now))) 702 return (error); 703 #ifdef KTRACE 704 if (KTRPOINT(p, KTR_STRUCT)) 705 ktrabstimespec(p, tsp); 706 #endif 707 708 if (timespeccmp(tsp, &now, <=)) { 709 /* already passed: still do the unlock */ 710 if ((error = thrsleep_unlock(lock))) 711 return (error); 712 return (EWOULDBLOCK); 713 } 714 715 timespecsub(tsp, &now, tsp); 716 nsecs = MIN(TIMESPEC_TO_NSEC(tsp), MAXTSLP); 717 } 718 719 if (ident == -1) { 720 queue = &thrsleep_queue; 721 qlock = &thrsleep_lock; 722 } else { 723 queue = &p->p_p->ps_tslpqueue; 724 qlock = &p->p_p->ps_lock; 725 } 726 727 /* Interlock with wakeup. */ 728 entry.tslp_ident = ident; 729 rw_enter_write(qlock); 730 TAILQ_INSERT_TAIL(queue, &entry, tslp_link); 731 rw_exit_write(qlock); 732 733 error = thrsleep_unlock(lock); 734 735 if (error == 0 && SCARG(uap, abort) != NULL) 736 error = copyin(SCARG(uap, abort), &abort, sizeof(abort)); 737 738 rw_enter_write(qlock); 739 if (error != 0) 740 goto out; 741 if (abort != 0) { 742 error = EINTR; 743 goto out; 744 } 745 if (entry.tslp_ident != 0) { 746 error = rwsleep_nsec(&entry, qlock, PWAIT|PCATCH, "thrsleep", 747 nsecs); 748 } 749 750 out: 751 if (entry.tslp_ident != 0) 752 TAILQ_REMOVE(queue, &entry, tslp_link); 753 rw_exit_write(qlock); 754 755 if (error == ERESTART) 756 error = ECANCELED; 757 758 return (error); 759 760 } 761 762 int 763 sys___thrsleep(struct proc *p, void *v, register_t *retval) 764 { 765 struct sys___thrsleep_args /* { 766 syscallarg(const volatile void *) ident; 767 syscallarg(clockid_t) clock_id; 768 syscallarg(struct timespec *) tp; 769 syscallarg(void *) lock; 770 syscallarg(const int *) abort; 771 } */ *uap = v; 772 struct timespec ts; 773 int error; 774 775 if (SCARG(uap, tp) != NULL) { 776 if ((error = copyin(SCARG(uap, tp), &ts, sizeof(ts)))) { 777 *retval = error; 778 return 0; 779 } 780 if (!timespecisvalid(&ts)) { 781 *retval = EINVAL; 782 return 0; 783 } 784 SCARG(uap, tp) = &ts; 785 } 786 787 *retval = thrsleep(p, uap); 788 return 0; 789 } 790 791 int 792 sys___thrwakeup(struct proc *p, void *v, register_t *retval) 793 { 794 struct sys___thrwakeup_args /* { 795 syscallarg(const volatile void *) ident; 796 syscallarg(int) n; 797 } */ *uap = v; 798 struct tslpentry *entry, *tmp; 799 struct tslpqueue *queue; 800 struct rwlock *qlock; 801 long ident = (long)SCARG(uap, ident); 802 int n = SCARG(uap, n); 803 int found = 0; 804 805 if (ident == 0) 806 *retval = EINVAL; 807 else { 808 if (ident == -1) { 809 queue = &thrsleep_queue; 810 qlock = &thrsleep_lock; 811 /* 812 * Wake up all waiters with ident -1. This is needed 813 * because ident -1 can be shared by multiple userspace 814 * lock state machines concurrently. The implementation 815 * has no way to direct the wakeup to a particular 816 * state machine. 817 */ 818 n = 0; 819 } else { 820 queue = &p->p_p->ps_tslpqueue; 821 qlock = &p->p_p->ps_lock; 822 } 823 824 rw_enter_write(qlock); 825 TAILQ_FOREACH_SAFE(entry, queue, tslp_link, tmp) { 826 if (entry->tslp_ident == ident) { 827 TAILQ_REMOVE(queue, entry, tslp_link); 828 entry->tslp_ident = 0; 829 wakeup_one(entry); 830 if (++found == n) 831 break; 832 } 833 } 834 rw_exit_write(qlock); 835 836 if (ident == -1) 837 *retval = 0; 838 else 839 *retval = found ? 0 : ESRCH; 840 } 841 842 return (0); 843 } 844 845 void 846 refcnt_init(struct refcnt *r) 847 { 848 r->refs = 1; 849 } 850 851 void 852 refcnt_take(struct refcnt *r) 853 { 854 #ifdef DIAGNOSTIC 855 u_int refcnt; 856 857 refcnt = atomic_inc_int_nv(&r->refs); 858 KASSERT(refcnt != 0); 859 #else 860 atomic_inc_int(&r->refs); 861 #endif 862 } 863 864 int 865 refcnt_rele(struct refcnt *r) 866 { 867 u_int refcnt; 868 869 refcnt = atomic_dec_int_nv(&r->refs); 870 KASSERT(refcnt != ~0); 871 872 return (refcnt == 0); 873 } 874 875 void 876 refcnt_rele_wake(struct refcnt *r) 877 { 878 if (refcnt_rele(r)) 879 wakeup_one(r); 880 } 881 882 void 883 refcnt_finalize(struct refcnt *r, const char *wmesg) 884 { 885 struct sleep_state sls; 886 u_int refcnt; 887 888 refcnt = atomic_dec_int_nv(&r->refs); 889 while (refcnt) { 890 sleep_setup(&sls, r, PWAIT, wmesg); 891 refcnt = r->refs; 892 sleep_finish(&sls, refcnt); 893 } 894 } 895 896 void 897 cond_init(struct cond *c) 898 { 899 c->c_wait = 1; 900 } 901 902 void 903 cond_signal(struct cond *c) 904 { 905 c->c_wait = 0; 906 907 wakeup_one(c); 908 } 909 910 void 911 cond_wait(struct cond *c, const char *wmesg) 912 { 913 struct sleep_state sls; 914 int wait; 915 916 wait = c->c_wait; 917 while (wait) { 918 sleep_setup(&sls, c, PWAIT, wmesg); 919 wait = c->c_wait; 920 sleep_finish(&sls, wait); 921 } 922 } 923