Lines Matching defs:ts
162 static int turnstile_adjust_thread(struct turnstile *ts,
164 static struct thread *turnstile_first_waiter(struct turnstile *ts);
165 static void turnstile_setowner(struct turnstile *ts, struct thread *owner);
178 propagate_unlock_ts(struct turnstile *top, struct turnstile *ts)
181 if (ts != top)
182 mtx_unlock_spin(&ts->ts_lock);
201 struct turnstile *ts, *top;
206 top = ts = td->td_blocked;
207 THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
215 td = ts->ts_owner;
222 propagate_unlock_ts(top, ts);
231 if (td->td_lock != &ts->ts_lock) {
233 propagate_unlock_ts(top, ts);
250 ts->ts_lockobj->lo_name);
291 ts->ts_lockobj->lo_name));
296 ts = td->td_blocked;
297 MPASS(ts != NULL);
298 THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
300 if (!turnstile_adjust_thread(ts, td)) {
301 propagate_unlock_ts(top, ts);
304 /* The thread lock is released as ts lock above. */
313 turnstile_adjust_thread(struct turnstile *ts, struct thread *td)
339 THREAD_LOCKPTR_BLOCKED_ASSERT(td, &ts->ts_lock);
351 TAILQ_REMOVE(&ts->ts_blocked[queue], td, td_lockq);
352 TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq) {
359 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
366 td->td_tid, ts->ts_lockobj, ts->ts_lockobj->lo_name);
370 td->td_tid, td1->td_tid, ts->ts_lockobj,
371 ts->ts_lockobj->lo_name);
445 struct turnstile *ts;
452 ts = td->td_blocked;
453 MPASS(ts != NULL);
454 THREAD_LOCKPTR_BLOCKED_ASSERT(td, &ts->ts_lock);
455 mtx_assert(&ts->ts_lock, MA_OWNED);
458 if (!turnstile_adjust_thread(ts, td))
468 if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) &&
478 turnstile_setowner(struct turnstile *ts, struct thread *owner)
482 MPASS(ts->ts_owner == NULL);
489 ts->ts_owner = owner;
490 LIST_INSERT_HEAD(&owner->td_contested, ts, ts_link);
500 struct turnstile *ts;
502 ts = mem;
503 MPASS(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]));
504 MPASS(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]));
505 MPASS(TAILQ_EMPTY(&ts->ts_pending));
515 struct turnstile *ts;
518 ts = mem;
519 TAILQ_INIT(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]);
520 TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]);
521 TAILQ_INIT(&ts->ts_pending);
522 LIST_INIT(&ts->ts_free);
523 mtx_init(&ts->ts_lock, "turnstile lock", NULL, MTX_SPIN);
530 struct turnstile *ts;
532 ts = mem;
533 mtx_destroy(&ts->ts_lock);
550 turnstile_free(struct turnstile *ts)
553 uma_zfree(turnstile_zone, ts);
572 struct turnstile *ts;
576 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
577 if (ts->ts_lockobj == lock) {
578 mtx_lock_spin(&ts->ts_lock);
579 return (ts);
582 ts = curthread->td_turnstile;
583 MPASS(ts != NULL);
584 mtx_lock_spin(&ts->ts_lock);
585 KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer"));
586 ts->ts_lockobj = lock;
588 return (ts);
592 turnstile_lock(struct turnstile *ts, struct lock_object **lockp,
598 if ((lock = ts->ts_lockobj) == NULL)
602 mtx_lock_spin(&ts->ts_lock);
603 if (__predict_false(lock != ts->ts_lockobj)) {
605 mtx_unlock_spin(&ts->ts_lock);
609 *tdp = ts->ts_owner;
614 turnstile_unlock(struct turnstile *ts, struct lock_object *lock)
618 mtx_assert(&ts->ts_lock, MA_OWNED);
619 mtx_unlock_spin(&ts->ts_lock);
620 if (ts == curthread->td_turnstile)
621 ts->ts_lockobj = NULL;
627 turnstile_assert(struct turnstile *ts)
629 MPASS(ts->ts_lockobj == NULL);
633 turnstile_cancel(struct turnstile *ts)
638 mtx_assert(&ts->ts_lock, MA_OWNED);
640 mtx_unlock_spin(&ts->ts_lock);
641 lock = ts->ts_lockobj;
642 if (ts == curthread->td_turnstile)
643 ts->ts_lockobj = NULL;
657 struct turnstile *ts;
661 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
662 if (ts->ts_lockobj == lock) {
663 mtx_lock_spin(&ts->ts_lock);
664 return (ts);
686 turnstile_first_waiter(struct turnstile *ts)
690 std = TAILQ_FIRST(&ts->ts_blocked[TS_SHARED_QUEUE]);
691 xtd = TAILQ_FIRST(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]);
702 turnstile_claim(struct turnstile *ts)
707 mtx_assert(&ts->ts_lock, MA_OWNED);
708 MPASS(ts != curthread->td_turnstile);
712 turnstile_setowner(ts, owner);
715 td = turnstile_first_waiter(ts);
718 THREAD_LOCKPTR_BLOCKED_ASSERT(td, &ts->ts_lock);
727 tc = TC_LOOKUP(ts->ts_lockobj);
728 mtx_unlock_spin(&ts->ts_lock);
739 turnstile_wait(struct turnstile *ts, struct thread *owner, int queue)
746 mtx_assert(&ts->ts_lock, MA_OWNED);
756 tc = TC_LOOKUP(ts->ts_lockobj);
758 if (ts == td->td_turnstile) {
767 LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash);
768 KASSERT(TAILQ_EMPTY(&ts->ts_pending),
770 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]),
772 KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]),
774 KASSERT(LIST_EMPTY(&ts->ts_free),
776 MPASS(ts->ts_lockobj != NULL);
778 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
779 turnstile_setowner(ts, owner);
782 TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq)
789 TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
790 MPASS(owner == ts->ts_owner);
793 LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash);
796 thread_lock_set(td, &ts->ts_lock);
800 lock = ts->ts_lockobj;
802 td->td_blocked = ts;
815 THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
828 turnstile_signal(struct turnstile *ts, int queue)
834 MPASS(ts != NULL);
835 mtx_assert(&ts->ts_lock, MA_OWNED);
837 MPASS(ts->ts_owner == curthread || ts->ts_owner == NULL);
844 td = TAILQ_FIRST(&ts->ts_blocked[queue]);
847 TAILQ_REMOVE(&ts->ts_blocked[queue], td, td_lockq);
849 TAILQ_INSERT_TAIL(&ts->ts_pending, td, td_lockq);
856 empty = TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) &&
857 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]);
859 tc = TC_LOOKUP(ts->ts_lockobj);
861 MPASS(LIST_EMPTY(&ts->ts_free));
866 ts = LIST_FIRST(&ts->ts_free);
867 MPASS(ts != NULL);
868 LIST_REMOVE(ts, ts_hash);
869 td->td_turnstile = ts;
879 turnstile_broadcast(struct turnstile *ts, int queue)
885 MPASS(ts != NULL);
886 mtx_assert(&ts->ts_lock, MA_OWNED);
888 MPASS(ts->ts_owner == curthread || ts->ts_owner == NULL);
893 tc = TC_LOOKUP(ts->ts_lockobj);
901 TAILQ_CONCAT(&ts->ts_pending, &ts->ts_blocked[queue], td_lockq);
908 TAILQ_FOREACH(td, &ts->ts_pending, td_lockq) {
909 if (LIST_EMPTY(&ts->ts_free)) {
911 ts1 = ts;
916 ts1 = LIST_FIRST(&ts->ts_free);
947 turnstile_unpend(struct turnstile *ts)
953 MPASS(ts != NULL);
954 mtx_assert(&ts->ts_lock, MA_OWNED);
955 MPASS(ts->ts_owner == curthread || ts->ts_owner == NULL);
956 MPASS(!TAILQ_EMPTY(&ts->ts_pending));
963 TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq);
965 if (TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) &&
966 TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]))
967 ts->ts_lockobj = NULL;
984 if (ts->ts_owner != NULL) {
985 ts->ts_owner = NULL;
986 LIST_REMOVE(ts, ts_link);
1004 THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock);
1017 mtx_unlock_spin(&ts->ts_lock);
1025 turnstile_disown(struct turnstile *ts)
1030 MPASS(ts != NULL);
1031 mtx_assert(&ts->ts_lock, MA_OWNED);
1032 MPASS(ts->ts_owner == curthread);
1033 MPASS(TAILQ_EMPTY(&ts->ts_pending));
1034 MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) ||
1035 !TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]));
1044 ts->ts_owner = NULL;
1045 LIST_REMOVE(ts, ts_link);
1055 mtx_unlock_spin(&ts->ts_lock);
1067 turnstile_head(struct turnstile *ts, int queue)
1071 MPASS(ts != NULL);
1073 mtx_assert(&ts->ts_lock, MA_OWNED);
1075 return (TAILQ_FIRST(&ts->ts_blocked[queue]));
1082 turnstile_empty(struct turnstile *ts, int queue)
1086 MPASS(ts != NULL);
1088 mtx_assert(&ts->ts_lock, MA_OWNED);
1090 return (TAILQ_EMPTY(&ts->ts_blocked[queue]));
1120 struct turnstile *ts;
1133 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
1134 if (ts->ts_lockobj == lock)
1142 LIST_FOREACH(ts, &turnstile_chains[i].tc_turnstiles, ts_hash) {
1143 if (ts == (struct turnstile *)addr)
1150 lock = ts->ts_lockobj;
1153 if (ts->ts_owner)
1154 print_thread(ts->ts_owner, "Lock Owner: ");
1157 print_queue(&ts->ts_blocked[TS_SHARED_QUEUE], "Shared Waiters", "\t");
1158 print_queue(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE], "Exclusive Waiters",
1160 print_queue(&ts->ts_pending, "Pending Threads", "\t");
1173 struct turnstile *ts;
1202 ts = td->td_blocked;
1203 lock = ts->ts_lockobj;
1207 if (ts->ts_owner == NULL)
1209 td = ts->ts_owner;
1267 static void print_waiters(struct turnstile *ts, int indent);
1272 struct turnstile *ts;
1280 LIST_FOREACH(ts, &td->td_contested, ts_link)
1281 print_waiters(ts, indent + 1);
1285 print_waiters(struct turnstile *ts, int indent)
1294 lock = ts->ts_lockobj;
1299 TAILQ_FOREACH(td, &ts->ts_blocked[TS_EXCLUSIVE_QUEUE], td_lockq)
1301 TAILQ_FOREACH(td, &ts->ts_blocked[TS_SHARED_QUEUE], td_lockq)
1303 TAILQ_FOREACH(td, &ts->ts_pending, td_lockq)
1312 struct turnstile *ts;
1318 LIST_FOREACH(ts, &tc->tc_turnstiles, ts_hash)
1319 if (ts->ts_lockobj == lock)
1321 if (ts == NULL) {
1326 print_waiters(ts, 0);