Lines Matching defs:uq
357 struct umtx_q *uq;
359 uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO);
360 uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX,
362 TAILQ_INIT(&uq->uq_spare_queue->head);
363 TAILQ_INIT(&uq->uq_pi_contested);
364 uq->uq_inherited_pri = PRI_MAX;
365 return (uq);
369 umtxq_free(struct umtx_q *uq)
372 MPASS(uq->uq_spare_queue != NULL);
373 free(uq->uq_spare_queue, M_UMTX);
374 free(uq, M_UMTX);
469 umtxq_insert_queue(struct umtx_q *uq, int q)
474 uc = umtxq_getchain(&uq->uq_key);
476 KASSERT((uq->uq_flags & UQF_UMTXQ) == 0, ("umtx_q is already on queue"));
477 uh = umtxq_queue_lookup(&uq->uq_key, q);
479 LIST_INSERT_HEAD(&uc->uc_spare_queue, uq->uq_spare_queue, link);
481 uh = uq->uq_spare_queue;
482 uh->key = uq->uq_key;
493 uq->uq_spare_queue = NULL;
495 TAILQ_INSERT_TAIL(&uh->head, uq, uq_link);
497 uq->uq_flags |= UQF_UMTXQ;
498 uq->uq_cur_queue = uh;
503 umtxq_remove_queue(struct umtx_q *uq, int q)
508 uc = umtxq_getchain(&uq->uq_key);
510 if (uq->uq_flags & UQF_UMTXQ) {
511 uh = uq->uq_cur_queue;
512 TAILQ_REMOVE(&uh->head, uq, uq_link);
514 uq->uq_flags &= ~UQF_UMTXQ;
527 uq->uq_spare_queue = uh;
528 uq->uq_cur_queue = NULL;
573 struct umtx_q *uq, *uq_temp;
581 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
582 if ((uq->uq_bitset & bitset) == 0)
584 umtxq_remove_queue(uq, UMTX_SHARED_QUEUE);
585 wakeup_one(uq);
600 struct umtx_q *uq;
607 while ((uq = TAILQ_FIRST(&uh->head)) != NULL) {
608 umtxq_remove_queue(uq, q);
609 wakeup(uq);
621 umtxq_signal_thread(struct umtx_q *uq)
624 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
625 umtxq_remove(uq);
626 wakeup(uq);
641 struct umtx_q *uq, *uq_temp;
650 TAILQ_FOREACH_SAFE(uq, &uh->head, uq_link, uq_temp) {
652 umtxq_remove(uq);
653 wakeup_one(uq);
655 umtxq_remove(uq);
656 uq->uq_key = *key2;
657 umtxq_insert(uq);
812 umtxq_sleep(struct umtx_q *uq, const char *wmesg,
819 uc = umtxq_getchain(&uq->uq_key);
822 if (!(uq->uq_flags & UQF_UMTXQ)) {
834 error = msleep_sbt(uq, &uc->uc_lock, PCATCH | PDROP, wmesg,
836 uc = umtxq_getchain(&uq->uq_key);
915 struct umtx_q *uq;
920 uq = td->td_umtxq;
970 AUTO_SHARE, &uq->uq_key)) != 0)
973 umtxq_lock(&uq->uq_key);
974 umtxq_busy(&uq->uq_key);
975 umtxq_insert(uq);
976 umtxq_unbusy(&uq->uq_key);
977 umtxq_unlock(&uq->uq_key);
989 umtxq_lock(&uq->uq_key);
990 umtxq_remove(uq);
991 umtxq_unlock(&uq->uq_key);
992 umtx_key_release(&uq->uq_key);
1001 umtxq_lock(&uq->uq_key);
1003 error = umtxq_sleep(uq, "umtx", timeout == NULL ? NULL :
1005 umtxq_remove(uq);
1006 umtxq_unlock(&uq->uq_key);
1007 umtx_key_release(&uq->uq_key);
1096 struct umtx_q *uq;
1101 uq = td->td_umtxq;
1151 AUTO_SHARE, &uq->uq_key)) != 0)
1154 umtxq_lock(&uq->uq_key);
1155 umtxq_busy(&uq->uq_key);
1156 umtxq_insert(uq);
1157 umtxq_unbusy(&uq->uq_key);
1158 umtxq_unlock(&uq->uq_key);
1170 umtxq_lock(&uq->uq_key);
1171 umtxq_remove(uq);
1172 umtxq_unlock(&uq->uq_key);
1173 umtx_key_release(&uq->uq_key);
1182 umtxq_lock(&uq->uq_key);
1184 error = umtxq_sleep(uq, "umtx", timeout == NULL ?
1186 umtxq_remove(uq);
1187 umtxq_unlock(&uq->uq_key);
1188 umtx_key_release(&uq->uq_key);
1277 struct umtx_q *uq;
1282 uq = td->td_umtxq;
1284 is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0)
1290 umtxq_lock(&uq->uq_key);
1291 umtxq_insert(uq);
1292 umtxq_unlock(&uq->uq_key);
1304 umtxq_lock(&uq->uq_key);
1307 error = umtxq_sleep(uq, "uwait", timeout == NULL ?
1309 if ((uq->uq_flags & UQF_UMTXQ) == 0)
1312 umtxq_remove(uq);
1313 } else if ((uq->uq_flags & UQF_UMTXQ) != 0) {
1314 umtxq_remove(uq);
1316 umtxq_unlock(&uq->uq_key);
1317 umtx_key_release(&uq->uq_key);
1350 struct umtx_q *uq;
1355 uq = td->td_umtxq;
1462 GET_SHARE(flags), &uq->uq_key)) != 0)
1465 umtxq_lock(&uq->uq_key);
1466 umtxq_busy(&uq->uq_key);
1467 umtxq_insert(uq);
1468 umtxq_unlock(&uq->uq_key);
1481 umtxq_lock(&uq->uq_key);
1482 umtxq_remove(uq);
1483 umtxq_unbusy(&uq->uq_key);
1484 umtxq_unlock(&uq->uq_key);
1485 umtx_key_release(&uq->uq_key);
1501 umtxq_lock(&uq->uq_key);
1502 umtxq_unbusy(&uq->uq_key);
1504 error = umtxq_sleep(uq, "umtxn", timeout == NULL ?
1506 umtxq_remove(uq);
1507 umtxq_unlock(&uq->uq_key);
1508 umtx_key_release(&uq->uq_key);
1765 struct umtx_q *uq, *uq1, *uq2;
1772 uq = td->td_umtxq;
1779 uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq);
1780 uq2 = TAILQ_NEXT(uq, uq_lockq);
1787 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
1796 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
1798 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
1851 struct umtx_q *uq;
1857 uq = td->td_umtxq;
1858 pi = uq->uq_pi_blocked;
1884 uq = td->td_umtxq;
1885 pi = uq->uq_pi_blocked;
1900 struct umtx_q *uq, *uq_owner;
1913 uq = TAILQ_FIRST(&pi2->pi_blocked);
1914 if (uq != NULL) {
1915 if (pri > UPRI(uq->uq_thread))
1916 pri = UPRI(uq->uq_thread);
1963 struct umtx_q *uq;
1980 uq = TAILQ_FIRST(&pi->pi_blocked);
1981 if (uq != NULL) {
1982 pri = UPRI(uq->uq_thread);
1999 struct umtx_q *uq;
2002 uq = td->td_umtxq;
2007 pi = uq->uq_pi_blocked;
2019 umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner,
2031 td = uq->uq_thread;
2033 UMTXQ_LOCKED_ASSERT(umtxq_getchain(&uq->uq_key));
2035 umtxq_insert(uq);
2055 TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq);
2057 TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq);
2059 uq->uq_pi_blocked = pi;
2065 umtxq_unbusy(&uq->uq_key);
2067 error = umtxq_sleep(uq, wmesg, timo);
2068 umtxq_remove(uq);
2071 uq->uq_pi_blocked = NULL;
2075 TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
2078 umtxq_unlock(&uq->uq_key);
2226 struct umtx_q *uq;
2232 uq = td->td_umtxq;
2236 &uq->uq_key)) != 0)
2242 umtxq_lock(&uq->uq_key);
2243 pi = umtx_pi_lookup(&uq->uq_key);
2247 umtxq_unlock(&uq->uq_key);
2249 umtxq_lock(&uq->uq_key);
2250 pi = umtx_pi_lookup(&uq->uq_key);
2257 new_pi->pi_key = uq->uq_key;
2263 umtxq_unlock(&uq->uq_key);
2338 umtxq_lock(&uq->uq_key);
2339 umtxq_busy(&uq->uq_key);
2341 umtxq_unbusy(&uq->uq_key);
2342 umtxq_unlock(&uq->uq_key);
2375 umtxq_lock(&uq->uq_key);
2376 umtxq_busy(&uq->uq_key);
2377 umtxq_unlock(&uq->uq_key);
2390 umtxq_unbusy_unlocked(&uq->uq_key);
2395 umtxq_unbusy_unlocked(&uq->uq_key);
2409 umtxq_lock(&uq->uq_key);
2413 error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
2424 umtxq_lock(&uq->uq_key);
2426 umtxq_unlock(&uq->uq_key);
2428 umtx_key_release(&uq->uq_key);
2523 struct umtx_q *uq, *uq2;
2531 uq = td->td_umtxq;
2534 &uq->uq_key)) != 0)
2542 old_inherited_pri = uq->uq_inherited_pri;
2543 umtxq_lock(&uq->uq_key);
2544 umtxq_busy(&uq->uq_key);
2545 umtxq_unlock(&uq->uq_key);
2565 if (new_pri < uq->uq_inherited_pri) {
2566 uq->uq_inherited_pri = new_pri;
2612 umtxq_unbusy_unlocked(&uq->uq_key);
2631 umtxq_lock(&uq->uq_key);
2632 umtxq_insert(uq);
2633 umtxq_unbusy(&uq->uq_key);
2634 error = umtxq_sleep(uq, "umtxpp", timeout == NULL ?
2636 umtxq_remove(uq);
2637 umtxq_unlock(&uq->uq_key);
2640 uq->uq_inherited_pri = old_inherited_pri;
2642 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2649 if (pri > uq->uq_inherited_pri)
2650 pri = uq->uq_inherited_pri;
2659 uq->uq_inherited_pri = old_inherited_pri;
2661 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2668 if (pri > uq->uq_inherited_pri)
2669 pri = uq->uq_inherited_pri;
2677 umtxq_unbusy_unlocked(&uq->uq_key);
2678 umtx_key_release(&uq->uq_key);
2689 struct umtx_q *uq, *uq2;
2696 uq = td->td_umtxq;
2749 uq->uq_inherited_pri = new_inherited_pri;
2751 TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) {
2758 if (pri > uq->uq_inherited_pri)
2759 pri = uq->uq_inherited_pri;
2773 struct umtx_q *uq;
2785 uq = td->td_umtxq;
2788 &uq->uq_key)) != 0)
2791 umtxq_lock(&uq->uq_key);
2792 umtxq_busy(&uq->uq_key);
2793 umtxq_unlock(&uq->uq_key);
2842 umtxq_lock(&uq->uq_key);
2843 umtxq_insert(uq);
2844 umtxq_unbusy(&uq->uq_key);
2845 error = umtxq_sleep(uq, "umtxpp", NULL);
2846 umtxq_remove(uq);
2847 umtxq_unlock(&uq->uq_key);
2849 umtxq_lock(&uq->uq_key);
2851 umtxq_signal(&uq->uq_key, INT_MAX);
2852 umtxq_unbusy(&uq->uq_key);
2853 umtxq_unlock(&uq->uq_key);
2854 umtx_key_release(&uq->uq_key);
2930 struct umtx_q *uq;
2934 uq = td->td_umtxq;
2938 error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key);
2945 umtx_key_release(&uq->uq_key);
2951 umtx_key_release(&uq->uq_key);
2958 umtxq_lock(&uq->uq_key);
2959 umtxq_busy(&uq->uq_key);
2960 umtxq_insert(uq);
2961 umtxq_unlock(&uq->uq_key);
2971 umtxq_lock(&uq->uq_key);
2972 umtxq_remove(uq);
2973 umtxq_unbusy(&uq->uq_key);
2978 umtxq_unbusy_unlocked(&uq->uq_key);
2986 umtxq_lock(&uq->uq_key);
2988 error = umtxq_sleep(uq, "ucond", timeout == NULL ?
2992 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3000 umtxq_busy(&uq->uq_key);
3001 if ((uq->uq_flags & UQF_UMTXQ) != 0) {
3002 int oldlen = uq->uq_cur_queue->length;
3003 umtxq_remove(uq);
3005 umtxq_unlock(&uq->uq_key);
3009 umtxq_lock(&uq->uq_key);
3012 umtxq_unbusy(&uq->uq_key);
3017 umtxq_unlock(&uq->uq_key);
3018 umtx_key_release(&uq->uq_key);
3087 struct umtx_q *uq;
3093 uq = td->td_umtxq;
3097 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3111 umtx_key_release(&uq->uq_key);
3119 umtx_key_release(&uq->uq_key);
3125 umtx_key_release(&uq->uq_key);
3130 umtx_key_release(&uq->uq_key);
3143 umtxq_lock(&uq->uq_key);
3144 umtxq_busy(&uq->uq_key);
3145 umtxq_unlock(&uq->uq_key);
3174 umtxq_unbusy_unlocked(&uq->uq_key);
3180 umtxq_unbusy_unlocked(&uq->uq_key);
3198 umtxq_unbusy_unlocked(&uq->uq_key);
3204 umtxq_lock(&uq->uq_key);
3205 umtxq_insert(uq);
3206 umtxq_unbusy(&uq->uq_key);
3208 error = umtxq_sleep(uq, "urdlck", timeout == NULL ?
3211 umtxq_busy(&uq->uq_key);
3212 umtxq_remove(uq);
3213 umtxq_unlock(&uq->uq_key);
3230 umtxq_unbusy_unlocked(&uq->uq_key);
3237 umtxq_unbusy_unlocked(&uq->uq_key);
3262 umtxq_unbusy_unlocked(&uq->uq_key);
3266 umtx_key_release(&uq->uq_key);
3276 struct umtx_q *uq;
3283 uq = td->td_umtxq;
3287 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3298 umtx_key_release(&uq->uq_key);
3306 umtx_key_release(&uq->uq_key);
3311 umtx_key_release(&uq->uq_key);
3324 umtxq_lock(&uq->uq_key);
3325 umtxq_busy(&uq->uq_key);
3326 umtxq_signal_queue(&uq->uq_key, INT_MAX,
3328 umtxq_unbusy(&uq->uq_key);
3329 umtxq_unlock(&uq->uq_key);
3336 umtxq_lock(&uq->uq_key);
3337 umtxq_busy(&uq->uq_key);
3338 umtxq_unlock(&uq->uq_key);
3367 umtxq_unbusy_unlocked(&uq->uq_key);
3373 umtxq_unbusy_unlocked(&uq->uq_key);
3386 umtxq_unbusy_unlocked(&uq->uq_key);
3393 umtxq_lock(&uq->uq_key);
3394 umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3395 umtxq_unbusy(&uq->uq_key);
3397 error = umtxq_sleep(uq, "uwrlck", timeout == NULL ?
3400 umtxq_busy(&uq->uq_key);
3401 umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
3402 umtxq_unlock(&uq->uq_key);
3418 umtxq_unbusy_unlocked(&uq->uq_key);
3425 umtxq_unbusy_unlocked(&uq->uq_key);
3456 umtxq_unbusy_unlocked(&uq->uq_key);
3463 umtxq_unbusy_unlocked(&uq->uq_key);
3466 umtx_key_release(&uq->uq_key);
3475 struct umtx_q *uq;
3480 uq = td->td_umtxq;
3484 error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key);
3559 umtxq_lock(&uq->uq_key);
3560 umtxq_busy(&uq->uq_key);
3561 umtxq_signal_queue(&uq->uq_key, count, q);
3562 umtxq_unbusy(&uq->uq_key);
3563 umtxq_unlock(&uq->uq_key);
3566 umtx_key_release(&uq->uq_key);
3575 struct umtx_q *uq;
3579 uq = td->td_umtxq;
3583 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3591 umtxq_lock(&uq->uq_key);
3592 umtxq_busy(&uq->uq_key);
3593 umtxq_insert(uq);
3594 umtxq_unlock(&uq->uq_key);
3601 umtxq_lock(&uq->uq_key);
3602 umtxq_unbusy(&uq->uq_key);
3603 umtxq_remove(uq);
3604 umtxq_unlock(&uq->uq_key);
3620 umtxq_lock(&uq->uq_key);
3621 umtxq_unbusy(&uq->uq_key);
3623 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3625 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3628 umtxq_remove(uq);
3634 umtxq_unlock(&uq->uq_key);
3636 umtx_key_release(&uq->uq_key);
3684 struct umtx_q *uq;
3688 uq = td->td_umtxq;
3694 error = umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), &uq->uq_key);
3697 umtxq_lock(&uq->uq_key);
3698 umtxq_busy(&uq->uq_key);
3699 umtxq_insert(uq);
3700 umtxq_unlock(&uq->uq_key);
3703 umtxq_lock(&uq->uq_key);
3704 umtxq_unbusy(&uq->uq_key);
3705 umtxq_remove(uq);
3706 umtxq_unlock(&uq->uq_key);
3707 umtx_key_release(&uq->uq_key);
3712 umtxq_lock(&uq->uq_key);
3713 umtxq_unbusy(&uq->uq_key);
3714 umtxq_remove(uq);
3715 umtxq_unlock(&uq->uq_key);
3716 umtx_key_release(&uq->uq_key);
3724 umtxq_lock(&uq->uq_key);
3725 umtxq_unbusy(&uq->uq_key);
3726 umtxq_remove(uq);
3727 umtxq_unlock(&uq->uq_key);
3728 umtx_key_release(&uq->uq_key);
3736 umtxq_lock(&uq->uq_key);
3737 umtxq_unbusy(&uq->uq_key);
3739 error = umtxq_sleep(uq, "usem", timeout == NULL ? NULL : &timo);
3741 if ((uq->uq_flags & UQF_UMTXQ) == 0)
3744 umtxq_remove(uq);
3757 umtxq_unlock(&uq->uq_key);
3758 umtx_key_release(&uq->uq_key);
5068 struct umtx_q *uq;
5070 uq = td->td_umtxq;
5071 uq->uq_inherited_pri = PRI_MAX;
5073 KASSERT(uq->uq_flags == 0, ("uq_flags != 0"));
5074 KASSERT(uq->uq_thread == td, ("uq_thread != td"));
5075 KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL"));
5076 KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty"));
5211 struct umtx_q *uq;
5219 uq = td->td_umtxq;
5220 if (uq != NULL) {
5221 if (uq->uq_inherited_pri != PRI_MAX ||
5222 !TAILQ_EMPTY(&uq->uq_pi_contested)) {
5224 uq->uq_inherited_pri = PRI_MAX;
5225 while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) {
5227 TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);