Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
75 #include <machine/cpu.h>
90 #define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED)
96 mtx_assert(&uc->uc_lock, MA_OWNED); \
97 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy")); \
104 * Don't propagate time-sharing priority, there is a security reason,
105 * a user can simply introduce PI-mutex, let thread A lock the mutex,
109 * if it is using 100%CPU, this is unfair to other processes.
112 #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\
113 (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\
114 PRI_MAX_TIMESHARE : (td)->td_user_pri)
120 #define UMTX_SHIFTS (__WORD_BIT - 9)
226 mtx_lock(&uc->uc_lock);
227 tot += uc->max_length;
228 mtx_unlock(&uc->uc_lock);
238 mtx_lock(&uc->uc_lock);
239 whole = uc->max_length * 100;
240 mtx_unlock(&uc->uc_lock);
297 if (error != 0 || req->newptr == NULL)
304 mtx_lock(&uc->uc_lock);
305 uc->length = 0;
306 uc->max_length = 0;
307 mtx_unlock(&uc->uc_lock);
360 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
362 TAILQ_INIT(&uq->uq_spare_queue->head);
363 TAILQ_INIT(&uq->uq_pi_contested);
364 uq->uq_inherited_pri = PRI_MAX;
372 MPASS(uq->uq_spare_queue != NULL);
373 free(uq->uq_spare_queue, M_UMTX);
382 n = (uintptr_t)key->info.both.a + key->info.both.b;
383 key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS;
390 if (key->type <= TYPE_SEM)
391 return (&umtxq_chains[1][key->hash]);
392 return (&umtxq_chains[0][key->hash]);
405 mtx_assert(&uc->uc_lock, MA_OWNED);
406 if (uc->uc_busy) {
412 while (uc->uc_busy && --count > 0)
418 while (uc->uc_busy) {
419 uc->uc_waiters++;
420 msleep(uc, &uc->uc_lock, 0, "umtxqb", 0);
421 uc->uc_waiters--;
424 uc->uc_busy = 1;
436 mtx_assert(&uc->uc_lock, MA_OWNED);
437 KASSERT(uc->uc_busy != 0, ("not busy"));
438 uc->uc_busy = 0;
439 if (uc->uc_waiters)
460 LIST_FOREACH(uh, &uc->uc_queue[q], link) {
461 if (umtx_key_match(&uh->key, key))
474 uc = umtxq_getchain(&uq->uq_key);
476 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
477 uh = umtxq_queue_lookup(&uq->uq_key, q);
479 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
481 uh = uq->uq_spare_queue;
482 uh->key = uq->uq_key;
483 LIST_INSERT_HEAD(&uc->uc_queue[q], uh, link);
485 uc->length++;
486 if (uc->length > uc->max_length) {
487 uc->max_length = uc->length;
488 if (uc->max_length > max_length)
489 max_length = uc->max_length;
493 uq->uq_spare_queue = NULL;
495 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
496 uh->length++;
497 uq->uq_flags |= UQF_UMTXQ;
498 uq->uq_cur_queue = uh;
508 uc = umtxq_getchain(&uq->uq_key);
510 if (uq->uq_flags & UQF_UMTXQ) {
511 uh = uq->uq_cur_queue;
512 TAILQ_REMOVE(&uh->head, uq, uq_link);
513 uh->length--;
514 uq->uq_flags &= ~UQF_UMTXQ;
515 if (TAILQ_EMPTY(&uh->head)) {
516 KASSERT(uh->length == 0,
519 uc->length--;
523 uh = LIST_FIRST(&uc->uc_spare_queue);
527 uq->uq_spare_queue = uh;
528 uq->uq_cur_queue = NULL;
543 return (uh->length);
560 *first = TAILQ_FIRST(&uh->head);
561 return (uh->length);
567 * Wake up threads waiting on an userland object by a bit mask.
581 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
582 if ((uq->uq_bitset & bitset) == 0)
593 * Wake up threads waiting on an userland object.
607 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
618 * Wake up specified thread.
624 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
630 * Wake up a maximum of n_wake threads that are waiting on an userland
650 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
656 uq->uq_key = *key2;
658 if (ret - n_wake == n_requeue)
679 timo->clockid = clockid;
681 timo->is_abs_real = false;
682 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
683 timespecadd(&timo->cur, timeout, &timo->end);
685 timo->end = *timeout;
686 timo->is_abs_real = clockid == CLOCK_REALTIME ||
698 umtx_abs_timeout_init(timo, umtxtime->_clockid,
699 (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout);
707 mint = curproc->p_umtx_min_timeout;
723 switch (timo->clockid) {
736 timespec2bintime(&timo->end, &bt);
737 switch (timo->clockid) {
758 * avoid firing multiple timer events in non-periodic
759 * timer mode.
761 switch (timo->clockid) {
767 *sbt += tc_tick_sbt - rem;
772 *sbt += SBT_1S - rem;
784 kern_clock_gettime(curthread, timo->clockid, &timo->cur);
785 if (timespeccmp(&timo->end, &timo->cur, <=))
787 timespecsub(&timo->end, &timo->cur, &tts);
819 uc = umtxq_getchain(&uq->uq_key);
822 if (!(uq->uq_flags & UQF_UMTXQ)) {
827 if (timo->is_abs_real)
828 curthread->td_rtcgen =
834 error = msleep_sbt(uq, &uc->uc_lock, PCATCH | PDROP, wmesg,
836 uc = umtxq_getchain(&uq->uq_key);
837 mtx_lock(&uc->uc_lock);
846 curthread->td_rtcgen = 0;
863 key->type = type;
865 key->shared = 0;
866 key->info.private.vs = td->td_proc->p_vmspace;
867 key->info.private.addr = (uintptr_t)addr;
870 map = &td->td_proc->p_vmspace->vm_map;
872 &entry, &key->info.shared.object, &pindex, &prot,
879 VM_INHERIT_SHARE == entry->inheritance)) {
880 key->shared = 1;
881 key->info.shared.offset = (vm_offset_t)addr -
882 entry->start + entry->offset;
883 vm_object_reference(key->info.shared.object);
885 key->shared = 0;
886 key->info.private.vs = td->td_proc->p_vmspace;
887 key->info.private.addr = (uintptr_t)addr;
902 if (key->shared)
903 vm_object_deallocate(key->info.shared.object);
920 uq = td->td_umtxq;
932 owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id);
939 if (owner == -1)
944 owner = casuword(&umtx->u_owner,
951 if (owner == -1)
970 AUTO_SHARE, &uq->uq_key)) != 0)
973 umtxq_lock(&uq->uq_key);
974 umtxq_busy(&uq->uq_key);
976 umtxq_unbusy(&uq->uq_key);
977 umtxq_unlock(&uq->uq_key);
985 old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED);
988 if (old == -1) {
989 umtxq_lock(&uq->uq_key);
991 umtxq_unlock(&uq->uq_key);
992 umtx_key_release(&uq->uq_key);
1001 umtxq_lock(&uq->uq_key);
1006 umtxq_unlock(&uq->uq_key);
1007 umtx_key_release(&uq->uq_key);
1018 /* Timed-locking is not restarted. */
1040 owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner));
1041 if (owner == -1)
1049 old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED);
1050 if (old == -1)
1072 old = casuword(&umtx->u_owner, owner,
1079 if (old == -1)
1101 uq = td->td_umtxq;
1121 if (owner == -1)
1132 if (owner == -1)
1151 AUTO_SHARE, &uq->uq_key)) != 0)
1154 umtxq_lock(&uq->uq_key);
1155 umtxq_busy(&uq->uq_key);
1157 umtxq_unbusy(&uq->uq_key);
1158 umtxq_unlock(&uq->uq_key);
1169 if (old == -1) {
1170 umtxq_lock(&uq->uq_key);
1172 umtxq_unlock(&uq->uq_key);
1173 umtx_key_release(&uq->uq_key);
1182 umtxq_lock(&uq->uq_key);
1187 umtxq_unlock(&uq->uq_key);
1188 umtx_key_release(&uq->uq_key);
1199 /* Timed-locking is not restarted. */
1222 if (owner == -1)
1231 if (old == -1)
1260 if (old == -1)
1282 uq = td->td_umtxq;
1284 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1290 umtxq_lock(&uq->uq_key);
1292 umtxq_unlock(&uq->uq_key);
1304 umtxq_lock(&uq->uq_key);
1309 if ((uq->uq_flags & UQF_UMTXQ) == 0)
1313 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
1316 umtxq_unlock(&uq->uq_key);
1317 umtx_key_release(&uq->uq_key);
1324 * Wake up threads sleeping on the specified address.
1354 id = td->td_tid;
1355 uq = td->td_umtxq;
1365 rv = fueword32(&m->m_owner, &owner);
1366 if (rv == -1)
1382 rv = casueword32(&m->m_owner,
1385 if (rv == -1)
1404 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED,
1407 if (rv == -1)
1422 rv = casueword32(&m->m_owner,
1426 if (rv == -1)
1462 GET_SHARE(flags), &uq->uq_key)) != 0)
1465 umtxq_lock(&uq->uq_key);
1466 umtxq_busy(&uq->uq_key);
1468 umtxq_unlock(&uq->uq_key);
1476 rv = casueword32(&m->m_owner, owner, &old,
1480 if (rv == -1 || rv == 1) {
1481 umtxq_lock(&uq->uq_key);
1483 umtxq_unbusy(&uq->uq_key);
1484 umtxq_unlock(&uq->uq_key);
1485 umtx_key_release(&uq->uq_key);
1486 if (rv == -1)
1501 umtxq_lock(&uq->uq_key);
1502 umtxq_unbusy(&uq->uq_key);
1507 umtxq_unlock(&uq->uq_key);
1508 umtx_key_release(&uq->uq_key);
1527 id = td->td_tid;
1533 error = fueword32(&m->m_owner, &owner);
1534 if (error == -1)
1542 error = casueword32(&m->m_owner, owner, &old, newlock);
1543 if (error == -1)
1572 error = casueword32(&m->m_owner, owner, &old, newlock);
1578 if (error == -1)
1592 * Check if the mutex is available and wake up a waiter,
1605 error = fueword32(&m->m_owner, &owner);
1606 if (error == -1)
1613 error = fueword32(&m->m_flags, &flags);
1614 if (error == -1)
1629 error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
1631 if (error == -1) {
1700 error = fueword32(&m->m_owner, &owner);
1701 if (error == -1)
1711 error = casueword32(&m->m_owner, owner, &old,
1713 if (error == -1) {
1746 TAILQ_INIT(&pi->pi_blocked);
1755 atomic_add_int(&umtx_pi_allocated, -1);
1772 uq = td->td_umtxq;
1781 if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) ||
1782 (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) {
1787 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1788 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
1789 td1 = uq1->uq_thread;
1790 MPASS(td1->td_proc->p_magic == P_MAGIC);
1796 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1808 if (pi->pi_owner == NULL)
1810 uq_owner = pi->pi_owner->td_umtxq;
1813 return (uq_owner->uq_pi_blocked);
1817 * Floyd's Cycle-Finding Algorithm.
1857 uq = td->td_umtxq;
1858 pi = uq->uq_pi_blocked;
1865 td = pi->pi_owner;
1869 MPASS(td->td_proc != NULL);
1870 MPASS(td->td_proc->p_magic == P_MAGIC);
1873 if (td->td_lend_user_pri > pri)
1884 uq = td->td_umtxq;
1885 pi = uq->uq_pi_blocked;
1908 while (pi != NULL && pi->pi_owner != NULL) {
1910 uq_owner = pi->pi_owner->td_umtxq;
1912 TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) {
1913 uq = TAILQ_FIRST(&pi2->pi_blocked);
1915 if (pri > UPRI(uq->uq_thread))
1916 pri = UPRI(uq->uq_thread);
1920 if (pri > uq_owner->uq_inherited_pri)
1921 pri = uq_owner->uq_inherited_pri;
1922 thread_lock(pi->pi_owner);
1923 sched_lend_user_prio(pi->pi_owner, pri);
1924 thread_unlock(pi->pi_owner);
1925 if ((pi = uq_owner->uq_pi_blocked) != NULL)
1926 umtx_pi_adjust_thread(pi, uq_owner->uq_thread);
1938 uq_owner = owner->td_umtxq;
1940 MPASS(pi->pi_owner == NULL);
1941 pi->pi_owner = owner;
1942 TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link);
1953 TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, pi, pi_link);
1954 pi->pi_owner = NULL;
1967 if (pi->pi_owner == owner) {
1972 if (pi->pi_owner != NULL) {
1980 uq = TAILQ_FIRST(&pi->pi_blocked);
1982 pri = UPRI(uq->uq_thread);
2002 uq = td->td_umtxq;
2007 pi = uq->uq_pi_blocked;
2028 uc = umtxq_getchain(&pi->pi_key);
2031 td = uq->uq_thread;
2033 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
2034 KASSERT(uc->uc_busy != 0, ("umtx chain is not busy"));
2037 if (pi->pi_owner == NULL) {
2039 td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid);
2042 if (pi->pi_owner == NULL)
2044 PROC_UNLOCK(td1->td_proc);
2048 TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
2049 pri = UPRI(uq1->uq_thread);
2057 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
2059 uq->uq_pi_blocked = pi;
2061 td->td_flags |= TDF_UPIBLOCKED;
2065 umtxq_unbusy(&uq->uq_key);
2071 uq->uq_pi_blocked = NULL;
2073 td->td_flags &= ~TDF_UPIBLOCKED;
2075 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
2078 umtxq_unlock(&uq->uq_key);
2090 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&pi->pi_key));
2091 pi->pi_refcount++;
2103 uc = umtxq_getchain(&pi->pi_key);
2105 KASSERT(pi->pi_refcount > 0, ("invalid reference count"));
2106 if (--pi->pi_refcount == 0) {
2108 if (pi->pi_owner != NULL)
2110 KASSERT(TAILQ_EMPTY(&pi->pi_blocked),
2113 TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
2130 TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) {
2131 if (umtx_key_match(&pi->pi_key, key)) {
2146 uc = umtxq_getchain(&pi->pi_key);
2148 TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink);
2165 pi = uq_first->uq_pi_blocked;
2167 if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) {
2172 uq_me = td->td_umtxq;
2173 if (pi->pi_owner == td)
2176 uq_first = TAILQ_FIRST(&pi->pi_blocked);
2178 (uq_first->uq_flags & UQF_UMTXQ) == 0) {
2182 TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
2183 uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
2185 if (pri > UPRI(uq_first2->uq_thread))
2186 pri = UPRI(uq_first2->uq_thread);
2210 if (pi->pi_owner == td)
2231 id = td->td_tid;
2232 uq = td->td_umtxq;
2236 &uq->uq_key)) != 0)
2242 umtxq_lock(&uq->uq_key);
2243 pi = umtx_pi_lookup(&uq->uq_key);
2247 umtxq_unlock(&uq->uq_key);
2249 umtxq_lock(&uq->uq_key);
2250 pi = umtx_pi_lookup(&uq->uq_key);
2257 new_pi->pi_key = uq->uq_key;
2263 umtxq_unlock(&uq->uq_key);
2273 rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id);
2275 if (rv == -1) {
2315 rv = casueword32(&m->m_owner, owner, &owner,
2318 if (rv == -1) {
2338 umtxq_lock(&uq->uq_key);
2339 umtxq_busy(&uq->uq_key);
2341 umtxq_unbusy(&uq->uq_key);
2342 umtxq_unlock(&uq->uq_key);
2350 (void)casuword32(&m->m_owner,
2375 umtxq_lock(&uq->uq_key);
2376 umtxq_busy(&uq->uq_key);
2377 umtxq_unlock(&uq->uq_key);
2385 rv = casueword32(&m->m_owner, owner, &old, owner |
2389 if (rv == -1) {
2390 umtxq_unbusy_unlocked(&uq->uq_key);
2395 umtxq_unbusy_unlocked(&uq->uq_key);
2409 umtxq_lock(&uq->uq_key);
2424 umtxq_lock(&uq->uq_key);
2426 umtxq_unlock(&uq->uq_key);
2428 umtx_key_release(&uq->uq_key);
2442 id = td->td_tid;
2448 error = fueword32(&m->m_owner, &owner);
2449 if (error == -1)
2459 error = casueword32(&m->m_owner, owner, &old, new_owner);
2460 if (error == -1)
2500 error = casueword32(&m->m_owner, owner, &old, new_owner);
2508 if (error == -1)
2530 id = td->td_tid;
2531 uq = td->td_umtxq;
2534 &uq->uq_key)) != 0)
2542 old_inherited_pri = uq->uq_inherited_pri;
2543 umtxq_lock(&uq->uq_key);
2544 umtxq_busy(&uq->uq_key);
2545 umtxq_unlock(&uq->uq_key);
2547 rv = fueword32(&m->m_ceilings[0], &ceiling);
2548 if (rv == -1) {
2552 ceiling = RTP_PRIO_MAX - ceiling;
2559 if (td->td_base_user_pri < new_pri) {
2565 if (new_pri < uq->uq_inherited_pri) {
2566 uq->uq_inherited_pri = new_pri;
2575 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2578 if (rv == -1) {
2589 rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD,
2591 if (rv == -1) {
2612 umtxq_unbusy_unlocked(&uq->uq_key);
2631 umtxq_lock(&uq->uq_key);
2633 umtxq_unbusy(&uq->uq_key);
2637 umtxq_unlock(&uq->uq_key);
2640 uq->uq_inherited_pri = old_inherited_pri;
2642 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2643 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2645 if (pri > UPRI(uq2->uq_thread))
2646 pri = UPRI(uq2->uq_thread);
2649 if (pri > uq->uq_inherited_pri)
2650 pri = uq->uq_inherited_pri;
2659 uq->uq_inherited_pri = old_inherited_pri;
2661 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2662 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2664 if (pri > UPRI(uq2->uq_thread))
2665 pri = UPRI(uq2->uq_thread);
2668 if (pri > uq->uq_inherited_pri)
2669 pri = uq->uq_inherited_pri;
2677 umtxq_unbusy_unlocked(&uq->uq_key);
2678 umtx_key_release(&uq->uq_key);
2695 id = td->td_tid;
2696 uq = td->td_umtxq;
2702 error = fueword32(&m->m_owner, &owner);
2703 if (error == -1)
2709 error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t));
2713 if (rceiling == -1)
2716 rceiling = RTP_PRIO_MAX - rceiling;
2735 error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) |
2744 if (error == -1)
2749 uq->uq_inherited_pri = new_inherited_pri;
2751 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2752 uq2 = TAILQ_FIRST(&pi->pi_blocked);
2754 if (pri > UPRI(uq2->uq_thread))
2755 pri = UPRI(uq2->uq_thread);
2758 if (pri > uq->uq_inherited_pri)
2759 pri = uq->uq_inherited_pri;
2777 error = fueword32(&m->m_flags, &flags);
2778 if (error == -1)
2784 id = td->td_tid;
2785 uq = td->td_umtxq;
2788 &uq->uq_key)) != 0)
2791 umtxq_lock(&uq->uq_key);
2792 umtxq_busy(&uq->uq_key);
2793 umtxq_unlock(&uq->uq_key);
2795 rv = fueword32(&m->m_ceilings[0], &save_ceiling);
2796 if (rv == -1) {
2801 rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner,
2803 if (rv == -1) {
2810 rv = suword32(&m->m_ceilings[0], ceiling);
2811 rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED);
2817 rv = suword32(&m->m_ceilings[0], ceiling);
2842 umtxq_lock(&uq->uq_key);
2844 umtxq_unbusy(&uq->uq_key);
2847 umtxq_unlock(&uq->uq_key);
2849 umtxq_lock(&uq->uq_key);
2851 umtxq_signal(&uq->uq_key, INT_MAX);
2852 umtxq_unbusy(&uq->uq_key);
2853 umtxq_unlock(&uq->uq_key);
2854 umtx_key_release(&uq->uq_key);
2872 error = fueword32(&m->m_flags, &flags);
2873 if (error == -1)
2893 /* Timed-locking is not restarted. */
2909 error = fueword32(&m->m_flags, &flags);
2910 if (error == -1)
2934 uq = td->td_umtxq;
2935 error = fueword32(&cv->c_flags, &flags);
2936 if (error == -1)
2938 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2943 error = fueword32(&cv->c_clockid, &clockid);
2944 if (error == -1) {
2945 umtx_key_release(&uq->uq_key);
2951 umtx_key_release(&uq->uq_key);
2958 umtxq_lock(&uq->uq_key);
2959 umtxq_busy(&uq->uq_key);
2961 umtxq_unlock(&uq->uq_key);
2967 error = fueword32(&cv->c_has_waiters, &hasw);
2969 error = suword32(&cv->c_has_waiters, 1);
2971 umtxq_lock(&uq->uq_key);
2973 umtxq_unbusy(&uq->uq_key);
2978 umtxq_unbusy_unlocked(&uq->uq_key);
2986 umtxq_lock(&uq->uq_key);
2992 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3000 umtxq_busy(&uq->uq_key);
3001 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
3002 int oldlen = uq->uq_cur_queue->length;
3005 umtxq_unlock(&uq->uq_key);
3006 if (suword32(&cv->c_has_waiters, 0) != 0 &&
3009 umtxq_lock(&uq->uq_key);
3012 umtxq_unbusy(&uq->uq_key);
3017 umtxq_unlock(&uq->uq_key);
3018 umtx_key_release(&uq->uq_key);
3032 error = fueword32(&cv->c_flags, &flags);
3033 if (error == -1)
3043 error = suword32(&cv->c_has_waiters, 0);
3044 if (error == -1)
3061 error = fueword32(&cv->c_flags, &flags);
3062 if (error == -1)
3072 error = suword32(&cv->c_has_waiters, 0);
3073 if (error == -1)
3093 uq = td->td_umtxq;
3094 error = fueword32(&rwlock->rw_flags, &flags);
3095 if (error == -1)
3097 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3109 rv = fueword32(&rwlock->rw_state, &state);
3110 if (rv == -1) {
3111 umtx_key_release(&uq->uq_key);
3119 umtx_key_release(&uq->uq_key);
3122 rv = casueword32(&rwlock->rw_state, state,
3124 if (rv == -1) {
3125 umtx_key_release(&uq->uq_key);
3130 umtx_key_release(&uq->uq_key);
3143 umtxq_lock(&uq->uq_key);
3144 umtxq_busy(&uq->uq_key);
3145 umtxq_unlock(&uq->uq_key);
3148 * re-read the state, in case it changed between the try-lock above
3151 rv = fueword32(&rwlock->rw_state, &state);
3152 if (rv == -1)
3158 rv = casueword32(&rwlock->rw_state, state,
3160 if (rv == -1) {
3174 umtxq_unbusy_unlocked(&uq->uq_key);
3180 umtxq_unbusy_unlocked(&uq->uq_key);
3192 rv = fueword32(&rwlock->rw_blocked_readers,
3195 rv = suword32(&rwlock->rw_blocked_readers,
3197 if (rv == -1) {
3198 umtxq_unbusy_unlocked(&uq->uq_key);
3204 umtxq_lock(&uq->uq_key);
3206 umtxq_unbusy(&uq->uq_key);
3211 umtxq_busy(&uq->uq_key);
3213 umtxq_unlock(&uq->uq_key);
3216 rv = fueword32(&rwlock->rw_state, &state);
3217 if (rv == -1) {
3224 rv = fueword32(&rwlock->rw_blocked_readers,
3227 rv = suword32(&rwlock->rw_blocked_readers,
3228 blocked_readers - 1);
3229 if (rv == -1) {
3230 umtxq_unbusy_unlocked(&uq->uq_key);
3235 rv = fueword32(&rwlock->rw_state, &state);
3236 if (rv == -1) {
3237 umtxq_unbusy_unlocked(&uq->uq_key);
3242 rv = casueword32(&rwlock->rw_state, state,
3244 if (rv == -1) {
3262 umtxq_unbusy_unlocked(&uq->uq_key);
3266 umtx_key_release(&uq->uq_key);
3283 uq = td->td_umtxq;
3284 error = fueword32(&rwlock->rw_flags, &flags);
3285 if (error == -1)
3287 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3296 rv = fueword32(&rwlock->rw_state, &state);
3297 if (rv == -1) {
3298 umtx_key_release(&uq->uq_key);
3303 rv = casueword32(&rwlock->rw_state, state,
3305 if (rv == -1) {
3306 umtx_key_release(&uq->uq_key);
3311 umtx_key_release(&uq->uq_key);
3324 umtxq_lock(&uq->uq_key);
3325 umtxq_busy(&uq->uq_key);
3326 umtxq_signal_queue(&uq->uq_key, INT_MAX,
3328 umtxq_unbusy(&uq->uq_key);
3329 umtxq_unlock(&uq->uq_key);
3336 umtxq_lock(&uq->uq_key);
3337 umtxq_busy(&uq->uq_key);
3338 umtxq_unlock(&uq->uq_key);
3341 * Re-read the state, in case it changed between the
3342 * try-lock above and the check below.
3344 rv = fueword32(&rwlock->rw_state, &state);
3345 if (rv == -1)
3351 rv = casueword32(&rwlock->rw_state, state,
3353 if (rv == -1) {
3367 umtxq_unbusy_unlocked(&uq->uq_key);
3373 umtxq_unbusy_unlocked(&uq->uq_key);
3380 rv = fueword32(&rwlock->rw_blocked_writers,
3383 rv = suword32(&rwlock->rw_blocked_writers,
3385 if (rv == -1) {
3386 umtxq_unbusy_unlocked(&uq->uq_key);
3393 umtxq_lock(&uq->uq_key);
3395 umtxq_unbusy(&uq->uq_key);
3400 umtxq_busy(&uq->uq_key);
3402 umtxq_unlock(&uq->uq_key);
3405 rv = fueword32(&rwlock->rw_state, &state);
3406 if (rv == -1) {
3412 rv = fueword32(&rwlock->rw_blocked_writers,
3415 rv = suword32(&rwlock->rw_blocked_writers,
3416 blocked_writers - 1);
3417 if (rv == -1) {
3418 umtxq_unbusy_unlocked(&uq->uq_key);
3423 rv = fueword32(&rwlock->rw_state, &state);
3424 if (rv == -1) {
3425 umtxq_unbusy_unlocked(&uq->uq_key);
3430 rv = casueword32(&rwlock->rw_state, state,
3432 if (rv == -1) {
3453 rv = fueword32(&rwlock->rw_blocked_readers,
3455 if (rv == -1) {
3456 umtxq_unbusy_unlocked(&uq->uq_key);
3463 umtxq_unbusy_unlocked(&uq->uq_key);
3466 umtx_key_release(&uq->uq_key);
3480 uq = td->td_umtxq;
3481 error = fueword32(&rwlock->rw_flags, &flags);
3482 if (error == -1)
3484 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3488 error = fueword32(&rwlock->rw_state, &state);
3489 if (error == -1) {
3495 rv = casueword32(&rwlock->rw_state, state,
3497 if (rv == -1) {
3515 rv = casueword32(&rwlock->rw_state, state,
3516 &oldstate, state - 1);
3517 if (rv == -1) {
3559 umtxq_lock(&uq->uq_key);
3560 umtxq_busy(&uq->uq_key);
3561 umtxq_signal_queue(&uq->uq_key, count, q);
3562 umtxq_unbusy(&uq->uq_key);
3563 umtxq_unlock(&uq->uq_key);
3566 umtx_key_release(&uq->uq_key);
3579 uq = td->td_umtxq;
3580 error = fueword32(&sem->_flags, &flags);
3581 if (error == -1)
3583 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3591 umtxq_lock(&uq->uq_key);
3592 umtxq_busy(&uq->uq_key);
3594 umtxq_unlock(&uq->uq_key);
3595 rv = casueword32(&sem->_has_waiters, 0, &count1, 1);
3596 if (rv != -1)
3597 rv1 = fueword32(&sem->_count, &count);
3598 if (rv == -1 || rv1 == -1 || count != 0 || (rv == 1 && count1 == 0)) {
3600 rv = suword32(&sem->_has_waiters, 0);
3601 umtxq_lock(&uq->uq_key);
3602 umtxq_unbusy(&uq->uq_key);
3604 umtxq_unlock(&uq->uq_key);
3605 if (rv == -1 || rv1 == -1) {
3620 umtxq_lock(&uq->uq_key);
3621 umtxq_unbusy(&uq->uq_key);
3625 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3629 /* A relative timeout cannot be restarted. */
3631 (timeout->_flags & UMTX_ABSTIME) == 0)
3634 umtxq_unlock(&uq->uq_key);
3636 umtx_key_release(&uq->uq_key);
3650 error = fueword32(&sem->_flags, &flags);
3651 if (error == -1)
3666 error = suword32(&sem->_has_waiters, 0);
3668 if (error == -1)
3688 uq = td->td_umtxq;
3689 flags = fuword32(&sem->_flags);
3694 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3697 umtxq_lock(&uq->uq_key);
3698 umtxq_busy(&uq->uq_key);
3700 umtxq_unlock(&uq->uq_key);
3701 rv = fueword32(&sem->_count, &count);
3702 if (rv == -1) {
3703 umtxq_lock(&uq->uq_key);
3704 umtxq_unbusy(&uq->uq_key);
3706 umtxq_unlock(&uq->uq_key);
3707 umtx_key_release(&uq->uq_key);
3712 umtxq_lock(&uq->uq_key);
3713 umtxq_unbusy(&uq->uq_key);
3715 umtxq_unlock(&uq->uq_key);
3716 umtx_key_release(&uq->uq_key);
3721 rv = casueword32(&sem->_count, 0, &count, USEM_HAS_WAITERS);
3724 umtxq_lock(&uq->uq_key);
3725 umtxq_unbusy(&uq->uq_key);
3727 umtxq_unlock(&uq->uq_key);
3728 umtx_key_release(&uq->uq_key);
3729 if (rv == -1)
3736 umtxq_lock(&uq->uq_key);
3737 umtxq_unbusy(&uq->uq_key);
3741 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3745 if (timeout != NULL && (timeout->_flags & UMTX_ABSTIME) == 0) {
3746 /* A relative timeout cannot be restarted. */
3753 &timeout->_timeout);
3757 umtxq_unlock(&uq->uq_key);
3758 umtx_key_release(&uq->uq_key);
3772 rv = fueword32(&sem->_flags, &flags);
3773 if (rv == -1)
3787 rv = fueword32(&sem->_count, &count);
3788 while (rv != -1 && count & USEM_HAS_WAITERS) {
3789 rv = casueword32(&sem->_count, count, &count,
3797 if (rv == -1)
3817 return (do_lock_umtx(td, uap->umtx, td->td_tid, 0));
3824 return (do_unlock_umtx(td, uap->umtx, td->td_tid));
3846 if (size <= sizeof(tp->_timeout)) {
3847 tp->_clockid = CLOCK_REALTIME;
3848 tp->_flags = 0;
3849 error = copyin(uaddr, &tp->_timeout, sizeof(tp->_timeout));
3854 if (!timespecvalid_interval(&tp->_timeout))
3874 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
3893 if (uap->uaddr2 == NULL)
3896 error = ops->copyin_timeout(uap->uaddr2, &timeout);
3902 if (ops->compat32)
3903 return (do_lock_umtx32(td, uap->obj, uap->val, ts));
3905 return (do_lock_umtx(td, uap->obj, uap->val, ts));
3913 if (ops->compat32)
3914 return (do_unlock_umtx32(td, uap->obj, uap->val));
3916 return (do_unlock_umtx(td, uap->obj, uap->val));
3936 if (uap->uaddr2 == NULL)
3939 error = ops->copyin_umtx_time(
3940 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3945 return (do_wait(td, uap->obj, uap->val, tm_p, ops->compat32, 0));
3955 if (uap->uaddr2 == NULL)
3958 error = ops->copyin_umtx_time(
3959 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3964 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0));
3974 if (uap->uaddr2 == NULL)
3977 error = ops->copyin_umtx_time(
3978 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
3983 return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1));
3991 return (kern_umtx_wake(td, uap->obj, uap->val, 0));
4001 upp = (char **)uap->obj;
4003 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4023 upp = (uint32_t *)uap->obj;
4025 for (count = uap->val, pos = 0; count > 0; count -= tocopy,
4045 if (ops->compat32)
4055 return (kern_umtx_wake(td, uap->obj, uap->val, 1));
4066 if (uap->uaddr2 == NULL)
4069 error = ops->copyin_umtx_time(
4070 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4075 return (do_lock_umutex(td, uap->obj, tm_p, 0));
4083 return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY));
4094 if (uap->uaddr2 == NULL)
4097 error = ops->copyin_umtx_time(
4098 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4103 return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT));
4111 return (do_wake_umutex(td, uap->obj));
4119 return (do_unlock_umutex(td, uap->obj, false));
4127 return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1));
4138 if (uap->uaddr2 == NULL)
4141 error = ops->copyin_timeout(uap->uaddr2, &timeout);
4146 return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val));
4154 return (do_cv_signal(td, uap->obj));
4162 return (do_cv_broadcast(td, uap->obj));
4173 if (uap->uaddr2 == NULL) {
4174 error = do_rw_rdlock(td, uap->obj, uap->val, 0);
4176 error = ops->copyin_umtx_time(uap->uaddr2,
4177 (size_t)uap->uaddr1, &timeout);
4180 error = do_rw_rdlock(td, uap->obj, uap->val, &timeout);
4193 if (uap->uaddr2 == NULL) {
4194 error = do_rw_wrlock(td, uap->obj, 0);
4196 error = ops->copyin_umtx_time(uap->uaddr2,
4197 (size_t)uap->uaddr1, &timeout);
4201 error = do_rw_wrlock(td, uap->obj, &timeout);
4211 return (do_rw_unlock(td, uap->obj));
4223 if (uap->uaddr2 == NULL)
4226 error = ops->copyin_umtx_time(
4227 uap->uaddr2, (size_t)uap->uaddr1, &timeout);
4232 return (do_sem_wait(td, uap->obj, tm_p));
4240 return (do_sem_wake(td, uap->obj));
4249 return (do_wake2_umutex(td, uap->obj, uap->val));
4261 if (uap->uaddr2 == NULL) {
4265 uasize = (size_t)uap->uaddr1;
4266 error = ops->copyin_umtx_time(uap->uaddr2, uasize, &timeout);
4271 error = do_sem2_wait(td, uap->obj, tm_p);
4272 if (error == EINTR && uap->uaddr2 != NULL &&
4274 uasize >= ops->umtx_time_sz + ops->timespec_sz) {
4275 error = ops->copyout_timeout(
4276 (void *)((uintptr_t)uap->uaddr2 + ops->umtx_time_sz),
4277 uasize - ops->umtx_time_sz, &timeout._timeout);
4291 return (do_sem2_wake(td, uap->obj));
4295 ((struct umtx_shm_obj_list *)(&(o)->umtx_data))
4353 KASSERT(key->shared, ("umtx_p_find_rg: private key"));
4355 reg_head = &umtx_shm_registry[key->hash];
4357 KASSERT(reg->ushm_key.shared,
4358 ("non-shared key on reg %p %d", reg, reg->ushm_key.shared));
4359 if (reg->ushm_key.info.shared.object ==
4360 key->info.shared.object &&
4361 reg->ushm_key.info.shared.offset ==
4362 key->info.shared.offset) {
4363 KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM"));
4364 KASSERT(reg->ushm_refcnt != 0,
4366 KASSERT((reg->ushm_flags & USHMF_LINKED) != 0,
4374 if (__predict_false(reg->ushm_refcnt == UINT_MAX))
4376 reg->ushm_refcnt++;
4402 chgumtxcnt(reg->ushm_cred->cr_ruidinfo, -1, 0);
4403 crfree(reg->ushm_cred);
4404 shm_drop(reg->ushm_obj);
4412 KASSERT(reg->ushm_refcnt != 0, ("ushm_reg %p refcnt 0", reg));
4415 if ((reg->ushm_flags & USHMF_LINKED) == 0)
4422 TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], reg,
4425 reg->ushm_flags &= ~USHMF_LINKED;
4428 reg->ushm_refcnt--;
4429 return (reg->ushm_refcnt == 0);
4441 * shared-memory VM object in presence of concurrent callers
4445 object = reg->ushm_obj->shm_object;
4508 cred = td->td_ucred;
4509 if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP)))
4511 shm = shm_alloc(td->td_ucred, O_RDWR, false);
4513 chgumtxcnt(cred->cr_ruidinfo, -1, 0);
4517 bcopy(key, ®->ushm_key, sizeof(*key));
4518 reg->ushm_obj = shm;
4519 reg->ushm_cred = crhold(cred);
4520 error = shm_dotruncate(reg->ushm_obj, PAGE_SIZE);
4526 /* Re-lookup as 'umtx_shm_lock' has been temporarily released. */
4541 TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link);
4542 LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg,
4544 reg->ushm_flags = USHMF_LINKED;
4551 reg->ushm_refcnt = 2;
4568 map = &td->td_proc->p_vmspace->vm_map;
4576 ret = (object->flags & OBJ_UMTXDEAD) != 0 ? ENOTTY : 0;
4609 KASSERT(key.shared == 1, ("non-shared key"));
4622 error = mac_posixshm_check_open(td->td_ucred,
4623 reg->ushm_obj, FFLAGS(O_RDWR));
4626 error = shm_access(reg->ushm_obj, td->td_ucred,
4632 shm_hold(reg->ushm_obj);
4633 finit(fp, FFLAGS(O_RDWR), DTYPE_SHM, reg->ushm_obj,
4635 td->td_retval[0] = fd;
4648 return (umtx_shm(td, uap->uaddr1, uap->val));
4658 if (ops->compat32) {
4659 if ((td->td_pflags2 & TDP2_COMPAT32RB) == 0 &&
4660 (td->td_rb_list != 0 || td->td_rbp_list != 0 ||
4661 td->td_rb_inact != 0))
4663 } else if ((td->td_pflags2 & TDP2_COMPAT32RB) != 0) {
4668 error = ops->copyin_robust_lists(uap->uaddr1, uap->val, &rb);
4672 if (ops->compat32)
4673 td->td_pflags2 |= TDP2_COMPAT32RB;
4675 td->td_rb_list = rb.robust_list_offset;
4676 td->td_rbp_list = rb.robust_priv_list_offset;
4677 td->td_rb_inact = rb.robust_inact_offset;
4688 val = sbttons(td->td_proc->p_umtx_min_timeout);
4689 if (ops->compat32) {
4691 error = copyout(&val1, uap->uaddr1, sizeof(val1));
4693 error = copyout(&val, uap->uaddr1, sizeof(val));
4702 if (uap->val < 0)
4704 td->td_proc->p_umtx_min_timeout = nstosbt(uap->val);
4710 * Provide the standard 32-bit definitions for x86, since native/compat32 use a
4711 * 32-bit time_t there. Other architectures just need the i386 definitions
4730 /* 32-bit architectures can emulate i386, so define these almost everywhere. */
4812 .tv_sec = tsp->tv_sec,
4813 .tv_nsec = tsp->tv_nsec,
4817 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4873 .tv_sec = tsp->tv_sec,
4874 .tv_nsec = tsp->tv_nsec,
4878 * Should be guaranteed by the caller, sz == uaddr1 - sizeof(_umtx_time)
4956 /* i386 can emulate other 32-bit archs, too! */
5002 if ((uap->op & (UMTX_OP__32BIT | UMTX_OP__I386)) != 0) {
5003 if ((uap->op & UMTX_OP__I386) != 0)
5010 if ((uap->op & UMTX_OP__I386) != 0)
5014 if ((uap->op & UMTX_OP__32BIT) != 0)
5017 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5018 uap->uaddr2, umtx_ops));
5027 return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL));
5034 return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid));
5042 return (kern__umtx_op(td, uap->obj, uap->op, uap->val, uap->uaddr1,
5043 uap->uaddr2, &umtx_native_ops32));
5051 td->td_umtxq = umtxq_alloc();
5052 td->td_umtxq->uq_thread = td;
5059 umtxq_free(td->td_umtxq);
5070 uq = td->td_umtxq;
5071 uq->uq_inherited_pri = PRI_MAX;
5073 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
5074 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
5075 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
5076 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
5092 KASSERT((p->p_flag & P_HADTHREADS) == 0 ||
5093 (p->p_flag & P_STOPPED_SINGLE) != 0,
5094 ("curproc must be single-threaded"));
5101 ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)),
5104 td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0;
5107 p->p_umtx_min_timeout = 0;
5151 *rb_list = m->m_rb_lnk;
5162 KASSERT(td->td_proc == curproc, ("need current vmspace"));
5170 if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid)
5197 td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb);
5201 td->td_proc->p_comm, td->td_proc->p_pid, name, error);
5219 uq = td->td_umtxq;
5221 if (uq->uq_inherited_pri != PRI_MAX ||
5222 !TAILQ_EMPTY(&uq->uq_pi_contested)) {
5224 uq->uq_inherited_pri = PRI_MAX;
5225 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
5226 pi->pi_owner = NULL;
5227 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
5234 compat32 = (td->td_pflags2 & TDP2_COMPAT32RB) != 0;
5235 td->td_pflags2 &= ~TDP2_COMPAT32RB;
5237 if (td->td_rb_inact == 0 && td->td_rb_list == 0 && td->td_rbp_list == 0)
5245 rb_inact = td->td_rb_inact;
5248 umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, "", compat32);
5249 umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv ", compat32);