xref: /openbsd-src/sys/kern/kern_synch.c (revision 46035553bfdd96e63c94e32da0210227ec2e3cf1)
1 /*	$OpenBSD: kern_synch.c,v 1.173 2020/12/24 01:16:14 cheloha Exp $	*/
2 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
3 
4 /*
5  * Copyright (c) 1982, 1986, 1990, 1991, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  * (c) UNIX System Laboratories, Inc.
8  * All or some portions of this file are derived from material licensed
9  * to the University of California by American Telephone and Telegraph
10  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
11  * the permission of UNIX System Laboratories, Inc.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  *
37  *	@(#)kern_synch.c	8.6 (Berkeley) 1/21/94
38  */
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/proc.h>
43 #include <sys/kernel.h>
44 #include <sys/signalvar.h>
45 #include <sys/resourcevar.h>
46 #include <sys/sched.h>
47 #include <sys/timeout.h>
48 #include <sys/mount.h>
49 #include <sys/syscallargs.h>
50 #include <sys/pool.h>
51 #include <sys/refcnt.h>
52 #include <sys/atomic.h>
53 #include <sys/smr.h>
54 #include <sys/witness.h>
55 #include <sys/tracepoint.h>
56 
57 #include <ddb/db_output.h>
58 
59 #include <machine/spinlock.h>
60 
61 #ifdef DIAGNOSTIC
62 #include <sys/syslog.h>
63 #endif
64 
65 #ifdef KTRACE
66 #include <sys/ktrace.h>
67 #endif
68 
69 int	thrsleep(struct proc *, struct sys___thrsleep_args *);
70 int	thrsleep_unlock(void *);
71 
72 /*
73  * We're only looking at 7 bits of the address; everything is
74  * aligned to 4, lots of things are aligned to greater powers
75  * of 2.  Shift right by 8, i.e. drop the bottom 256 worth.
76  */
77 #define TABLESIZE	128
78 #define LOOKUP(x)	(((long)(x) >> 8) & (TABLESIZE - 1))
79 TAILQ_HEAD(slpque,proc) slpque[TABLESIZE];
80 
81 void
82 sleep_queue_init(void)
83 {
84 	int i;
85 
86 	for (i = 0; i < TABLESIZE; i++)
87 		TAILQ_INIT(&slpque[i]);
88 }
89 
90 /*
91  * Global sleep channel for threads that do not want to
92  * receive wakeup(9) broadcasts.
93  */
94 int nowake;
95 
96 /*
97  * During autoconfiguration or after a panic, a sleep will simply
98  * lower the priority briefly to allow interrupts, then return.
99  * The priority to be used (safepri) is machine-dependent, thus this
100  * value is initialized and maintained in the machine-dependent layers.
101  * This priority will typically be 0, or the lowest priority
102  * that is safe for use on the interrupt stack; it can be made
103  * higher to block network software interrupts after panics.
104  */
105 extern int safepri;
106 
107 /*
108  * General sleep call.  Suspends the current process until a wakeup is
109  * performed on the specified identifier.  The process will then be made
110  * runnable with the specified priority.  Sleeps at most timo/hz seconds
111  * (0 means no timeout).  If pri includes PCATCH flag, signals are checked
112  * before and after sleeping, else signals are not checked.  Returns 0 if
113  * awakened, EWOULDBLOCK if the timeout expires.  If PCATCH is set and a
114  * signal needs to be delivered, ERESTART is returned if the current system
115  * call should be restarted if possible, and EINTR is returned if the system
116  * call should be interrupted by the signal (return EINTR).
117  */
118 int
119 tsleep(const volatile void *ident, int priority, const char *wmesg, int timo)
120 {
121 	struct sleep_state sls;
122 #ifdef MULTIPROCESSOR
123 	int hold_count;
124 #endif
125 
126 	KASSERT((priority & ~(PRIMASK | PCATCH)) == 0);
127 	KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0);
128 
129 #ifdef MULTIPROCESSOR
130 	KASSERT(timo || _kernel_lock_held());
131 #endif
132 
133 #ifdef DDB
134 	if (cold == 2)
135 		db_stack_dump();
136 #endif
137 	if (cold || panicstr) {
138 		int s;
139 		/*
140 		 * After a panic, or during autoconfiguration,
141 		 * just give interrupts a chance, then just return;
142 		 * don't run any other procs or panic below,
143 		 * in case this is the idle process and already asleep.
144 		 */
145 		s = splhigh();
146 		splx(safepri);
147 #ifdef MULTIPROCESSOR
148 		if (_kernel_lock_held()) {
149 			hold_count = __mp_release_all(&kernel_lock);
150 			__mp_acquire_count(&kernel_lock, hold_count);
151 		}
152 #endif
153 		splx(s);
154 		return (0);
155 	}
156 
157 	sleep_setup(&sls, ident, priority, wmesg);
158 	sleep_setup_timeout(&sls, timo);
159 	sleep_setup_signal(&sls);
160 
161 	return sleep_finish_all(&sls, 1);
162 }
163 
164 int
165 tsleep_nsec(const volatile void *ident, int priority, const char *wmesg,
166     uint64_t nsecs)
167 {
168 	uint64_t to_ticks;
169 
170 	if (nsecs == INFSLP)
171 		return tsleep(ident, priority, wmesg, 0);
172 #ifdef DIAGNOSTIC
173 	if (nsecs == 0) {
174 		log(LOG_WARNING,
175 		    "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n",
176 		    __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid,
177 		    wmesg);
178 	}
179 #endif
180 	/*
181 	 * We want to sleep at least nsecs nanoseconds worth of ticks.
182 	 *
183 	 *  - Clamp nsecs to prevent arithmetic overflow.
184 	 *
185 	 *  - Round nsecs up to account for any nanoseconds that do not
186 	 *    divide evenly into tick_nsec, otherwise we'll lose them to
187 	 *    integer division in the next step.  We add (tick_nsec - 1)
188 	 *    to keep from introducing a spurious tick if there are no
189 	 *    such nanoseconds, i.e. nsecs % tick_nsec == 0.
190 	 *
191 	 *  - Divide the rounded value to a count of ticks.  We divide
192 	 *    by (tick_nsec + 1) to discard the extra tick introduced if,
193 	 *    before rounding, nsecs % tick_nsec == 1.
194 	 *
195 	 *  - Finally, add a tick to the result.  We need to wait out
196 	 *    the current tick before we can begin counting our interval,
197 	 *    as we do not know how much time has elapsed since the
198 	 *    current tick began.
199 	 */
200 	nsecs = MIN(nsecs, UINT64_MAX - tick_nsec);
201 	to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1;
202 	if (to_ticks > INT_MAX)
203 		to_ticks = INT_MAX;
204 	return tsleep(ident, priority, wmesg, (int)to_ticks);
205 }
206 
207 /*
208  * Same as tsleep, but if we have a mutex provided, then once we've
209  * entered the sleep queue we drop the mutex. After sleeping we re-lock.
210  */
211 int
212 msleep(const volatile void *ident, struct mutex *mtx, int priority,
213     const char *wmesg, int timo)
214 {
215 	struct sleep_state sls;
216 	int error, spl;
217 #ifdef MULTIPROCESSOR
218 	int hold_count;
219 #endif
220 
221 	KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0);
222 	KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0);
223 	KASSERT(mtx != NULL);
224 
225 	if (priority & PCATCH)
226 		KERNEL_ASSERT_LOCKED();
227 
228 	if (cold || panicstr) {
229 		/*
230 		 * After a panic, or during autoconfiguration,
231 		 * just give interrupts a chance, then just return;
232 		 * don't run any other procs or panic below,
233 		 * in case this is the idle process and already asleep.
234 		 */
235 		spl = MUTEX_OLDIPL(mtx);
236 		MUTEX_OLDIPL(mtx) = safepri;
237 		mtx_leave(mtx);
238 #ifdef MULTIPROCESSOR
239 		if (_kernel_lock_held()) {
240 			hold_count = __mp_release_all(&kernel_lock);
241 			__mp_acquire_count(&kernel_lock, hold_count);
242 		}
243 #endif
244 		if ((priority & PNORELOCK) == 0) {
245 			mtx_enter(mtx);
246 			MUTEX_OLDIPL(mtx) = spl;
247 		} else
248 			splx(spl);
249 		return (0);
250 	}
251 
252 	sleep_setup(&sls, ident, priority, wmesg);
253 	sleep_setup_timeout(&sls, timo);
254 
255 	/* XXX - We need to make sure that the mutex doesn't
256 	 * unblock splsched. This can be made a bit more
257 	 * correct when the sched_lock is a mutex.
258 	 */
259 	spl = MUTEX_OLDIPL(mtx);
260 	MUTEX_OLDIPL(mtx) = splsched();
261 	mtx_leave(mtx);
262 	/* signal may stop the process, release mutex before that */
263 	sleep_setup_signal(&sls);
264 
265 	error = sleep_finish_all(&sls, 1);
266 
267 	if ((priority & PNORELOCK) == 0) {
268 		mtx_enter(mtx);
269 		MUTEX_OLDIPL(mtx) = spl; /* put the ipl back */
270 	} else
271 		splx(spl);
272 
273 	return error;
274 }
275 
276 int
277 msleep_nsec(const volatile void *ident, struct mutex *mtx, int priority,
278     const char *wmesg, uint64_t nsecs)
279 {
280 	uint64_t to_ticks;
281 
282 	if (nsecs == INFSLP)
283 		return msleep(ident, mtx, priority, wmesg, 0);
284 #ifdef DIAGNOSTIC
285 	if (nsecs == 0) {
286 		log(LOG_WARNING,
287 		    "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n",
288 		    __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid,
289 		    wmesg);
290 	}
291 #endif
292 	nsecs = MIN(nsecs, UINT64_MAX - tick_nsec);
293 	to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1;
294 	if (to_ticks > INT_MAX)
295 		to_ticks = INT_MAX;
296 	return msleep(ident, mtx, priority, wmesg, (int)to_ticks);
297 }
298 
299 /*
300  * Same as tsleep, but if we have a rwlock provided, then once we've
301  * entered the sleep queue we drop the it. After sleeping we re-lock.
302  */
303 int
304 rwsleep(const volatile void *ident, struct rwlock *rwl, int priority,
305     const char *wmesg, int timo)
306 {
307 	struct sleep_state sls;
308 	int error, status;
309 
310 	KASSERT((priority & ~(PRIMASK | PCATCH | PNORELOCK)) == 0);
311 	KASSERT(ident != &nowake || ISSET(priority, PCATCH) || timo != 0);
312 	rw_assert_anylock(rwl);
313 	status = rw_status(rwl);
314 
315 	sleep_setup(&sls, ident, priority, wmesg);
316 	sleep_setup_timeout(&sls, timo);
317 
318 	rw_exit(rwl);
319 	/* signal may stop the process, release rwlock before that */
320 	sleep_setup_signal(&sls);
321 
322 	error = sleep_finish_all(&sls, 1);
323 
324 	if ((priority & PNORELOCK) == 0)
325 		rw_enter(rwl, status);
326 
327 	return error;
328 }
329 
330 int
331 rwsleep_nsec(const volatile void *ident, struct rwlock *rwl, int priority,
332     const char *wmesg, uint64_t nsecs)
333 {
334 	uint64_t to_ticks;
335 
336 	if (nsecs == INFSLP)
337 		return rwsleep(ident, rwl, priority, wmesg, 0);
338 #ifdef DIAGNOSTIC
339 	if (nsecs == 0) {
340 		log(LOG_WARNING,
341 		    "%s: %s[%d]: %s: trying to sleep zero nanoseconds\n",
342 		    __func__, curproc->p_p->ps_comm, curproc->p_p->ps_pid,
343 		    wmesg);
344 	}
345 #endif
346 	nsecs = MIN(nsecs, UINT64_MAX - tick_nsec);
347 	to_ticks = (nsecs + tick_nsec - 1) / (tick_nsec + 1) + 1;
348 	if (to_ticks > INT_MAX)
349 		to_ticks = INT_MAX;
350 	return 	rwsleep(ident, rwl, priority, wmesg, (int)to_ticks);
351 }
352 
353 void
354 sleep_setup(struct sleep_state *sls, const volatile void *ident, int prio,
355     const char *wmesg)
356 {
357 	struct proc *p = curproc;
358 
359 #ifdef DIAGNOSTIC
360 	if (p->p_flag & P_CANTSLEEP)
361 		panic("sleep: %s failed insomnia", p->p_p->ps_comm);
362 	if (ident == NULL)
363 		panic("tsleep: no ident");
364 	if (p->p_stat != SONPROC)
365 		panic("tsleep: not SONPROC");
366 #endif
367 
368 	sls->sls_catch = prio & PCATCH;
369 	sls->sls_do_sleep = 1;
370 	sls->sls_locked = 0;
371 	sls->sls_sig = 0;
372 	sls->sls_unwind = 0;
373 	sls->sls_timeout = 0;
374 
375 	/*
376 	 * The kernel has to be locked for signal processing.
377 	 * This is done here and not in sleep_setup_signal() because
378 	 * KERNEL_LOCK() has to be taken before SCHED_LOCK().
379 	 */
380 	if (sls->sls_catch != 0) {
381 		KERNEL_LOCK();
382 		sls->sls_locked = 1;
383 	}
384 
385 	SCHED_LOCK(sls->sls_s);
386 
387 	TRACEPOINT(sched, sleep, NULL);
388 
389 	p->p_wchan = ident;
390 	p->p_wmesg = wmesg;
391 	p->p_slptime = 0;
392 	p->p_slppri = prio & PRIMASK;
393 	TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], p, p_runq);
394 }
395 
396 int
397 sleep_finish_all(struct sleep_state *sls, int do_sleep)
398 {
399 	int error, error1;
400 
401 	sleep_finish(sls, do_sleep);
402 	error1 = sleep_finish_timeout(sls);
403 	error = sleep_finish_signal(sls);
404 
405 	/* Signal errors are higher priority than timeouts. */
406 	if (error == 0 && error1 != 0)
407 		error = error1;
408 
409 	return error;
410 }
411 
412 void
413 sleep_finish(struct sleep_state *sls, int do_sleep)
414 {
415 	struct proc *p = curproc;
416 
417 	if (sls->sls_do_sleep && do_sleep) {
418 		p->p_stat = SSLEEP;
419 		p->p_ru.ru_nvcsw++;
420 		SCHED_ASSERT_LOCKED();
421 		mi_switch();
422 	} else if (!do_sleep) {
423 		unsleep(p);
424 	}
425 
426 #ifdef DIAGNOSTIC
427 	if (p->p_stat != SONPROC)
428 		panic("sleep_finish !SONPROC");
429 #endif
430 
431 	p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri;
432 	SCHED_UNLOCK(sls->sls_s);
433 
434 	/*
435 	 * Even though this belongs to the signal handling part of sleep,
436 	 * we need to clear it before the ktrace.
437 	 */
438 	atomic_clearbits_int(&p->p_flag, P_SINTR);
439 }
440 
441 void
442 sleep_setup_timeout(struct sleep_state *sls, int timo)
443 {
444 	struct proc *p = curproc;
445 
446 	KASSERT((p->p_flag & P_TIMEOUT) == 0);
447 
448 	if (timo) {
449 		sls->sls_timeout = 1;
450 		timeout_add(&p->p_sleep_to, timo);
451 	}
452 }
453 
454 int
455 sleep_finish_timeout(struct sleep_state *sls)
456 {
457 	struct proc *p = curproc;
458 
459 	if (sls->sls_timeout) {
460 		if (p->p_flag & P_TIMEOUT) {
461 			atomic_clearbits_int(&p->p_flag, P_TIMEOUT);
462 			return (EWOULDBLOCK);
463 		} else {
464 			/* This must not sleep. */
465 			timeout_del_barrier(&p->p_sleep_to);
466 			KASSERT((p->p_flag & P_TIMEOUT) == 0);
467 		}
468 	}
469 
470 	return (0);
471 }
472 
473 void
474 sleep_setup_signal(struct sleep_state *sls)
475 {
476 	struct proc *p = curproc;
477 
478 	if (sls->sls_catch == 0)
479 		return;
480 
481 	/* sleep_setup() has locked the kernel. */
482 	KERNEL_ASSERT_LOCKED();
483 
484 	/*
485 	 * We put ourselves on the sleep queue and start our timeout before
486 	 * calling single_thread_check or CURSIG, as we could stop there, and
487 	 * a wakeup or a SIGCONT (or both) could occur while we were stopped.
488 	 * A SIGCONT would cause us to be marked as SSLEEP without resuming us,
489 	 * thus we must be ready for sleep when CURSIG is called.  If the
490 	 * wakeup happens while we're stopped, p->p_wchan will be 0 upon
491 	 * return from single_thread_check or CURSIG.  In that case we should
492 	 * not go to sleep.  If single_thread_check returns an error we need
493 	 * to unwind immediately.  That's achieved by saving the return value
494 	 * in sls->sl_unwind and checking it later in sleep_finish_signal.
495 	 */
496 	atomic_setbits_int(&p->p_flag, P_SINTR);
497 	if ((sls->sls_unwind = single_thread_check(p, 1)) != 0 ||
498 	    (sls->sls_sig = CURSIG(p)) != 0) {
499 		unsleep(p);
500 		p->p_stat = SONPROC;
501 		sls->sls_do_sleep = 0;
502 	} else if (p->p_wchan == 0) {
503 		sls->sls_catch = 0;
504 		sls->sls_do_sleep = 0;
505 	}
506 }
507 
508 int
509 sleep_finish_signal(struct sleep_state *sls)
510 {
511 	struct proc *p = curproc;
512 	int error = 0;
513 
514 	if (sls->sls_catch != 0) {
515 		KERNEL_ASSERT_LOCKED();
516 
517 		if (sls->sls_unwind != 0 ||
518 		    (sls->sls_unwind = single_thread_check(p, 1)) != 0)
519 			error = sls->sls_unwind;
520 		else if (sls->sls_sig != 0 ||
521 		    (sls->sls_sig = CURSIG(p)) != 0) {
522 			if (p->p_p->ps_sigacts->ps_sigintr &
523 			    sigmask(sls->sls_sig))
524 				error = EINTR;
525 			else
526 				error = ERESTART;
527 		}
528 	}
529 
530 	if (sls->sls_locked)
531 		KERNEL_UNLOCK();
532 
533 	return (error);
534 }
535 
536 int
537 wakeup_proc(struct proc *p, const volatile void *chan)
538 {
539 	int s, awakened = 0;
540 
541 	SCHED_LOCK(s);
542 	if (p->p_wchan != NULL &&
543 	   ((chan == NULL) || (p->p_wchan == chan))) {
544 		awakened = 1;
545 		if (p->p_stat == SSLEEP)
546 			setrunnable(p);
547 		else
548 			unsleep(p);
549 	}
550 	SCHED_UNLOCK(s);
551 
552 	return awakened;
553 }
554 
555 /*
556  * Implement timeout for tsleep.
557  * If process hasn't been awakened (wchan non-zero),
558  * set timeout flag and undo the sleep.  If proc
559  * is stopped, just unsleep so it will remain stopped.
560  */
561 void
562 endtsleep(void *arg)
563 {
564 	struct proc *p = arg;
565 	int s;
566 
567 	SCHED_LOCK(s);
568 	if (wakeup_proc(p, NULL))
569 		atomic_setbits_int(&p->p_flag, P_TIMEOUT);
570 	SCHED_UNLOCK(s);
571 }
572 
573 /*
574  * Remove a process from its wait queue
575  */
576 void
577 unsleep(struct proc *p)
578 {
579 	SCHED_ASSERT_LOCKED();
580 
581 	if (p->p_wchan != NULL) {
582 		TAILQ_REMOVE(&slpque[LOOKUP(p->p_wchan)], p, p_runq);
583 		p->p_wchan = NULL;
584 		TRACEPOINT(sched, wakeup, p->p_tid, p->p_p->ps_pid);
585 	}
586 }
587 
588 /*
589  * Make a number of processes sleeping on the specified identifier runnable.
590  */
591 void
592 wakeup_n(const volatile void *ident, int n)
593 {
594 	struct slpque *qp;
595 	struct proc *p;
596 	struct proc *pnext;
597 	int s;
598 
599 	SCHED_LOCK(s);
600 	qp = &slpque[LOOKUP(ident)];
601 	for (p = TAILQ_FIRST(qp); p != NULL && n != 0; p = pnext) {
602 		pnext = TAILQ_NEXT(p, p_runq);
603 #ifdef DIAGNOSTIC
604 		/*
605 		 * If the rwlock passed to rwsleep() is contended, the
606 		 * CPU will end up calling wakeup() between sleep_setup()
607 		 * and sleep_finish().
608 		 */
609 		if (p == curproc) {
610 			KASSERT(p->p_stat == SONPROC);
611 			continue;
612 		}
613 		if (p->p_stat != SSLEEP && p->p_stat != SSTOP)
614 			panic("wakeup: p_stat is %d", (int)p->p_stat);
615 #endif
616 		if (wakeup_proc(p, ident))
617 			--n;
618 	}
619 	SCHED_UNLOCK(s);
620 }
621 
622 /*
623  * Make all processes sleeping on the specified identifier runnable.
624  */
625 void
626 wakeup(const volatile void *chan)
627 {
628 	wakeup_n(chan, -1);
629 }
630 
631 int
632 sys_sched_yield(struct proc *p, void *v, register_t *retval)
633 {
634 	struct proc *q;
635 	uint8_t newprio;
636 	int s;
637 
638 	SCHED_LOCK(s);
639 	/*
640 	 * If one of the threads of a multi-threaded process called
641 	 * sched_yield(2), drop its priority to ensure its siblings
642 	 * can make some progress.
643 	 */
644 	newprio = p->p_usrpri;
645 	SMR_TAILQ_FOREACH_LOCKED(q, &p->p_p->ps_threads, p_thr_link)
646 		newprio = max(newprio, q->p_runpri);
647 	setrunqueue(p->p_cpu, p, newprio);
648 	p->p_ru.ru_nvcsw++;
649 	mi_switch();
650 	SCHED_UNLOCK(s);
651 
652 	return (0);
653 }
654 
655 int
656 thrsleep_unlock(void *lock)
657 {
658 	static _atomic_lock_t unlocked = _ATOMIC_LOCK_UNLOCKED;
659 	_atomic_lock_t *atomiclock = lock;
660 
661 	if (!lock)
662 		return 0;
663 
664 	return copyout(&unlocked, atomiclock, sizeof(unlocked));
665 }
666 
667 struct tslpentry {
668 	TAILQ_ENTRY(tslpentry)	tslp_link;
669 	long			tslp_ident;
670 };
671 
672 /* thrsleep queue shared between processes */
673 static struct tslpqueue thrsleep_queue = TAILQ_HEAD_INITIALIZER(thrsleep_queue);
674 static struct rwlock thrsleep_lock = RWLOCK_INITIALIZER("thrsleeplk");
675 
676 int
677 thrsleep(struct proc *p, struct sys___thrsleep_args *v)
678 {
679 	struct sys___thrsleep_args /* {
680 		syscallarg(const volatile void *) ident;
681 		syscallarg(clockid_t) clock_id;
682 		syscallarg(const struct timespec *) tp;
683 		syscallarg(void *) lock;
684 		syscallarg(const int *) abort;
685 	} */ *uap = v;
686 	long ident = (long)SCARG(uap, ident);
687 	struct tslpentry entry;
688 	struct tslpqueue *queue;
689 	struct rwlock *qlock;
690 	struct timespec *tsp = (struct timespec *)SCARG(uap, tp);
691 	void *lock = SCARG(uap, lock);
692 	uint64_t nsecs = INFSLP;
693 	int abort = 0, error;
694 	clockid_t clock_id = SCARG(uap, clock_id);
695 
696 	if (ident == 0)
697 		return (EINVAL);
698 	if (tsp != NULL) {
699 		struct timespec now;
700 
701 		if ((error = clock_gettime(p, clock_id, &now)))
702 			return (error);
703 #ifdef KTRACE
704 		if (KTRPOINT(p, KTR_STRUCT))
705 			ktrabstimespec(p, tsp);
706 #endif
707 
708 		if (timespeccmp(tsp, &now, <=)) {
709 			/* already passed: still do the unlock */
710 			if ((error = thrsleep_unlock(lock)))
711 				return (error);
712 			return (EWOULDBLOCK);
713 		}
714 
715 		timespecsub(tsp, &now, tsp);
716 		nsecs = MIN(TIMESPEC_TO_NSEC(tsp), MAXTSLP);
717 	}
718 
719 	if (ident == -1) {
720 		queue = &thrsleep_queue;
721 		qlock = &thrsleep_lock;
722 	} else {
723 		queue = &p->p_p->ps_tslpqueue;
724 		qlock = &p->p_p->ps_lock;
725 	}
726 
727 	/* Interlock with wakeup. */
728 	entry.tslp_ident = ident;
729 	rw_enter_write(qlock);
730 	TAILQ_INSERT_TAIL(queue, &entry, tslp_link);
731 	rw_exit_write(qlock);
732 
733 	error = thrsleep_unlock(lock);
734 
735 	if (error == 0 && SCARG(uap, abort) != NULL)
736 		error = copyin(SCARG(uap, abort), &abort, sizeof(abort));
737 
738 	rw_enter_write(qlock);
739 	if (error != 0)
740 		goto out;
741 	if (abort != 0) {
742 		error = EINTR;
743 		goto out;
744 	}
745 	if (entry.tslp_ident != 0) {
746 		error = rwsleep_nsec(&entry, qlock, PWAIT|PCATCH, "thrsleep",
747 		    nsecs);
748 	}
749 
750 out:
751 	if (entry.tslp_ident != 0)
752 		TAILQ_REMOVE(queue, &entry, tslp_link);
753 	rw_exit_write(qlock);
754 
755 	if (error == ERESTART)
756 		error = ECANCELED;
757 
758 	return (error);
759 
760 }
761 
762 int
763 sys___thrsleep(struct proc *p, void *v, register_t *retval)
764 {
765 	struct sys___thrsleep_args /* {
766 		syscallarg(const volatile void *) ident;
767 		syscallarg(clockid_t) clock_id;
768 		syscallarg(struct timespec *) tp;
769 		syscallarg(void *) lock;
770 		syscallarg(const int *) abort;
771 	} */ *uap = v;
772 	struct timespec ts;
773 	int error;
774 
775 	if (SCARG(uap, tp) != NULL) {
776 		if ((error = copyin(SCARG(uap, tp), &ts, sizeof(ts)))) {
777 			*retval = error;
778 			return 0;
779 		}
780 		if (!timespecisvalid(&ts)) {
781 			*retval = EINVAL;
782 			return 0;
783 		}
784 		SCARG(uap, tp) = &ts;
785 	}
786 
787 	*retval = thrsleep(p, uap);
788 	return 0;
789 }
790 
791 int
792 sys___thrwakeup(struct proc *p, void *v, register_t *retval)
793 {
794 	struct sys___thrwakeup_args /* {
795 		syscallarg(const volatile void *) ident;
796 		syscallarg(int) n;
797 	} */ *uap = v;
798 	struct tslpentry *entry, *tmp;
799 	struct tslpqueue *queue;
800 	struct rwlock *qlock;
801 	long ident = (long)SCARG(uap, ident);
802 	int n = SCARG(uap, n);
803 	int found = 0;
804 
805 	if (ident == 0)
806 		*retval = EINVAL;
807 	else {
808 		if (ident == -1) {
809 			queue = &thrsleep_queue;
810 			qlock = &thrsleep_lock;
811 			/*
812 			 * Wake up all waiters with ident -1. This is needed
813 			 * because ident -1 can be shared by multiple userspace
814 			 * lock state machines concurrently. The implementation
815 			 * has no way to direct the wakeup to a particular
816 			 * state machine.
817 			 */
818 			n = 0;
819 		} else {
820 			queue = &p->p_p->ps_tslpqueue;
821 			qlock = &p->p_p->ps_lock;
822 		}
823 
824 		rw_enter_write(qlock);
825 		TAILQ_FOREACH_SAFE(entry, queue, tslp_link, tmp) {
826 			if (entry->tslp_ident == ident) {
827 				TAILQ_REMOVE(queue, entry, tslp_link);
828 				entry->tslp_ident = 0;
829 				wakeup_one(entry);
830 				if (++found == n)
831 					break;
832 			}
833 		}
834 		rw_exit_write(qlock);
835 
836 		if (ident == -1)
837 			*retval = 0;
838 		else
839 			*retval = found ? 0 : ESRCH;
840 	}
841 
842 	return (0);
843 }
844 
845 void
846 refcnt_init(struct refcnt *r)
847 {
848 	r->refs = 1;
849 }
850 
851 void
852 refcnt_take(struct refcnt *r)
853 {
854 #ifdef DIAGNOSTIC
855 	u_int refcnt;
856 
857 	refcnt = atomic_inc_int_nv(&r->refs);
858 	KASSERT(refcnt != 0);
859 #else
860 	atomic_inc_int(&r->refs);
861 #endif
862 }
863 
864 int
865 refcnt_rele(struct refcnt *r)
866 {
867 	u_int refcnt;
868 
869 	refcnt = atomic_dec_int_nv(&r->refs);
870 	KASSERT(refcnt != ~0);
871 
872 	return (refcnt == 0);
873 }
874 
875 void
876 refcnt_rele_wake(struct refcnt *r)
877 {
878 	if (refcnt_rele(r))
879 		wakeup_one(r);
880 }
881 
882 void
883 refcnt_finalize(struct refcnt *r, const char *wmesg)
884 {
885 	struct sleep_state sls;
886 	u_int refcnt;
887 
888 	refcnt = atomic_dec_int_nv(&r->refs);
889 	while (refcnt) {
890 		sleep_setup(&sls, r, PWAIT, wmesg);
891 		refcnt = r->refs;
892 		sleep_finish(&sls, refcnt);
893 	}
894 }
895 
896 void
897 cond_init(struct cond *c)
898 {
899 	c->c_wait = 1;
900 }
901 
902 void
903 cond_signal(struct cond *c)
904 {
905 	c->c_wait = 0;
906 
907 	wakeup_one(c);
908 }
909 
910 void
911 cond_wait(struct cond *c, const char *wmesg)
912 {
913 	struct sleep_state sls;
914 	int wait;
915 
916 	wait = c->c_wait;
917 	while (wait) {
918 		sleep_setup(&sls, c, PWAIT, wmesg);
919 		wait = c->c_wait;
920 		sleep_finish(&sls, wait);
921 	}
922 }
923