Lines Matching refs:count
111 uint64_t count; in lockmgr_shared() local
121 count = lkp->lk_count; in lockmgr_shared()
135 if ((count & LKC_XMASK) && lkp->lk_lockholder == td) { in lockmgr_shared()
155 while ((count & LKC_SHARED) && in lockmgr_shared()
156 (count & (LKC_EXREQ | LKC_UPREQ))) { in lockmgr_shared()
161 if (count & LKC_CANCEL) in lockmgr_shared()
174 count = atomic_fetchadd_long(&lkp->lk_count, 0); in lockmgr_shared()
176 if ((count & LKC_SHARED) && in lockmgr_shared()
177 (count & (LKC_EXREQ | LKC_UPREQ))) { in lockmgr_shared()
182 count = lkp->lk_count; in lockmgr_shared()
197 count = atomic_fetchadd_64(&lkp->lk_count, LKC_SCOUNT) + LKC_SCOUNT; in lockmgr_shared()
206 if ((count & (LKC_XMASK | LKC_EXREQ | in lockmgr_shared()
209 &count, count | LKC_SHARED)) { in lockmgr_shared()
216 (count & (LKC_XMASK | LKC_SHARED)) == 0) { in lockmgr_shared()
218 &count, count | LKC_SHARED)) { in lockmgr_shared()
224 if (count & LKC_SHARED) in lockmgr_shared()
234 if (count & LKC_CANCEL) { in lockmgr_shared()
268 count = lkp->lk_count; in lockmgr_shared()
269 if (count & LKC_SHARED) in lockmgr_shared()
272 count = atomic_fetchadd_64(&lkp->lk_count, 0); in lockmgr_shared()
286 uint64_t count; in lockmgr_exclusive() local
299 count = lkp->lk_count; in lockmgr_exclusive()
306 if ((count & LKC_XMASK) && lkp->lk_lockholder == td) { in lockmgr_exclusive()
312 count = atomic_fetchadd_64(&lkp->lk_count, 1) + 1; in lockmgr_exclusive()
313 KKASSERT((count & LKC_XMASK) > 1); in lockmgr_exclusive()
338 if ((count & (LKC_UPREQ | LKC_EXREQ | in lockmgr_exclusive()
340 ((count & LKC_SHARED) == 0 || in lockmgr_exclusive()
341 (count & LKC_SMASK) == 0)) { in lockmgr_exclusive()
342 ncount = (count + 1) & ~LKC_SHARED; in lockmgr_exclusive()
344 &count, ncount)) { in lockmgr_exclusive()
353 if (count & LKC_CANCEL) in lockmgr_exclusive()
365 if (count & (LKC_EXREQ | LKC_XMASK)) in lockmgr_exclusive()
366 ncount = count | LKC_EXREQ2; in lockmgr_exclusive()
368 ncount = count | LKC_EXREQ; in lockmgr_exclusive()
370 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in lockmgr_exclusive()
376 if ((count & (LKC_EXREQ | LKC_XMASK)) == 0) { in lockmgr_exclusive()
377 count = ncount; in lockmgr_exclusive()
383 count = lkp->lk_count; /* relod */ in lockmgr_exclusive()
415 if ((count & LKC_EXREQ) == 0) { in lockmgr_exclusive()
416 KKASSERT(count & LKC_XMASK); in lockmgr_exclusive()
427 if (count & LKC_CANCEL) { in lockmgr_exclusive()
477 count = lkp->lk_count; in lockmgr_exclusive()
479 if ((count & LKC_EXREQ) == 0) { in lockmgr_exclusive()
480 KKASSERT(count & LKC_XMASK); in lockmgr_exclusive()
486 count = atomic_fetchadd_64(&lkp->lk_count, 0); in lockmgr_exclusive()
500 uint64_t count; in lockmgr_downgrade() local
508 count = lkp->lk_count; in lockmgr_downgrade()
521 if (lkp->lk_lockholder != td || (count & LKC_XMASK) != 1) { in lockmgr_downgrade()
523 "%p/%p %016jx", lkp->lk_lockholder, td, count); in lockmgr_downgrade()
535 ncount = (count & ~(LKC_XMASK | LKC_EXREQ2)) + in lockmgr_downgrade()
536 ((count & LKC_XMASK) << LKC_SSHIFT); in lockmgr_downgrade()
539 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in lockmgr_downgrade()
545 if (count & (LKC_SMASK | LKC_EXREQ2)) in lockmgr_downgrade()
578 uint64_t count; in lockmgr_upgrade() local
590 count = lkp->lk_count; in lockmgr_upgrade()
597 if (count & LKC_XMASK) { in lockmgr_upgrade()
602 if ((count & LKC_SMASK) == 0) in lockmgr_upgrade()
616 if (count & LKC_UPREQ) { in lockmgr_upgrade()
631 if ((count & LKC_SMASK) == LKC_SCOUNT) { in lockmgr_upgrade()
635 ncount = (count - LKC_SCOUNT + 1) & ~LKC_SHARED; in lockmgr_upgrade()
636 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in lockmgr_upgrade()
653 ncount = (count - LKC_SCOUNT) | LKC_UPREQ; in lockmgr_upgrade()
654 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in lockmgr_upgrade()
655 count = ncount; in lockmgr_upgrade()
679 if ((count & LKC_UPREQ) == 0) { in lockmgr_upgrade()
680 KKASSERT((count & LKC_XMASK) == 1); in lockmgr_upgrade()
686 if (count & LKC_CANCEL) { in lockmgr_upgrade()
722 count = lkp->lk_count; in lockmgr_upgrade()
723 if ((count & LKC_UPREQ) == 0) { in lockmgr_upgrade()
724 KKASSERT((count & LKC_XMASK) == 1); in lockmgr_upgrade()
729 count = atomic_fetchadd_64(&lkp->lk_count, 0); in lockmgr_upgrade()
744 uint64_t count; in lockmgr_release() local
753 count = lkp->lk_count; in lockmgr_release()
767 if ((count & (LKC_SMASK | LKC_XMASK)) == 0) in lockmgr_release()
770 if (count & LKC_XMASK) { in lockmgr_release()
781 if ((count & (LKC_UPREQ | LKC_EXREQ | in lockmgr_release()
800 ncount = (count - 1); in lockmgr_release()
804 &count, ncount)) { in lockmgr_release()
805 if (count & (LKC_SMASK | LKC_EXREQ2)) in lockmgr_release()
814 } else if ((count & (LKC_UPREQ | LKC_XMASK)) == in lockmgr_release()
831 ncount = count & ~LKC_UPREQ; in lockmgr_release()
833 &count, ncount)) { in lockmgr_release()
842 } else if ((count & (LKC_EXREQ | LKC_XMASK)) == in lockmgr_release()
863 ncount = count & ~LKC_EXREQ; in lockmgr_release()
865 &count, ncount)) { in lockmgr_release()
880 count = in lockmgr_release()
892 KKASSERT((count & LKC_SHARED) && (count & LKC_SMASK)); in lockmgr_release()
893 if ((count & (LKC_EXREQ | LKC_UPREQ | LKC_SMASK)) == in lockmgr_release()
901 ncount = (count - LKC_SCOUNT) & ~LKC_CANCEL; in lockmgr_release()
903 &count, ncount)) { in lockmgr_release()
909 } else if ((count & (LKC_UPREQ | LKC_SMASK)) == in lockmgr_release()
920 ncount = (count - LKC_SCOUNT + 1) & in lockmgr_release()
923 &count, ncount)) { in lockmgr_release()
930 } else if ((count & (LKC_EXREQ | LKC_SMASK)) == in lockmgr_release()
939 ncount = (count - LKC_SCOUNT + 1) & in lockmgr_release()
943 &count, ncount)) { in lockmgr_release()
980 uint64_t count; in lockmgr_cancel_beg() local
982 count = lkp->lk_count; in lockmgr_cancel_beg()
986 KKASSERT((count & LKC_CANCEL) == 0); /* disallowed case */ in lockmgr_cancel_beg()
989 KKASSERT((count & (LKC_XMASK | LKC_SMASK)) != 0); in lockmgr_cancel_beg()
992 &count, count | LKC_CANCEL)) { in lockmgr_cancel_beg()
1004 if (count & (LKC_EXREQ | LKC_EXREQ2 | LKC_SMASK | LKC_UPREQ)) { in lockmgr_cancel_beg()
1042 uint64_t count; in undo_shreq() local
1045 count = atomic_fetchadd_64(&lkp->lk_count, -LKC_SCOUNT) - LKC_SCOUNT; in undo_shreq()
1046 while ((count & (LKC_EXREQ | LKC_UPREQ | LKC_CANCEL)) && in undo_shreq()
1047 (count & (LKC_SMASK | LKC_XMASK)) == 0) { in undo_shreq()
1059 if (count & LKC_UPREQ) { in undo_shreq()
1060 ncount = (count + 1) & ~(LKC_UPREQ | LKC_CANCEL | in undo_shreq()
1062 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_shreq()
1070 if (count & LKC_EXREQ) { in undo_shreq()
1071 ncount = (count + 1) & ~(LKC_EXREQ | LKC_EXREQ2 | in undo_shreq()
1073 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_shreq()
1081 if (count & LKC_CANCEL) { in undo_shreq()
1082 ncount = count & ~LKC_CANCEL; in undo_shreq()
1083 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_shreq()
1103 uint64_t count; in undo_exreq() local
1107 count = lkp->lk_count; in undo_exreq()
1113 if ((count & LKC_EXREQ) == 0) { in undo_exreq()
1119 if (count & LKC_XMASK) { in undo_exreq()
1130 ncount = count & ~(LKC_EXREQ | LKC_EXREQ2); in undo_exreq()
1131 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_exreq()
1132 if ((count & (LKC_EXREQ2 | LKC_UPREQ)) == in undo_exreq()
1141 } else if (count & LKC_UPREQ) { in undo_exreq()
1151 ncount = count & ~(LKC_EXREQ | LKC_EXREQ2); in undo_exreq()
1152 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_exreq()
1157 } else if ((count & LKC_SHARED) && (count & LKC_SMASK)) { in undo_exreq()
1169 ncount = count & ~(LKC_EXREQ | LKC_EXREQ2); in undo_exreq()
1170 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_exreq()
1187 ncount = (count + 1) & ~(LKC_EXREQ | LKC_EXREQ2); in undo_exreq()
1188 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_exreq()
1189 if (count & (LKC_EXREQ2 | LKC_SHARED)) in undo_exreq()
1212 uint64_t count; in undo_upreq() local
1216 count = lkp->lk_count; in undo_upreq()
1222 if ((count & LKC_UPREQ) == 0) { in undo_upreq()
1228 if (count & LKC_XMASK) { in undo_upreq()
1234 if (atomic_fcmpset_64(&lkp->lk_count, &count, in undo_upreq()
1235 count & ~LKC_UPREQ)) { in undo_upreq()
1240 } else if (count & LKC_EXREQ) { in undo_upreq()
1245 ncount = (count + 1); in undo_upreq()
1248 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_upreq()
1263 ncount = count & ~LKC_UPREQ; in undo_upreq()
1264 if (count & LKC_SMASK) in undo_upreq()
1267 if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { in undo_upreq()
1268 if ((count & LKC_SHARED) || in undo_upreq()
1327 uint64_t count __unused; in lockuninit()
1329 count = lkp->lk_count; in lockuninit()
1331 KKASSERT((count & (LKC_EXREQ | LKC_UPREQ)) == 0 && in lockuninit()
1332 ((count & LKC_SHARED) || (count & LKC_SMASK) == 0)); in lockuninit()
1342 uint64_t count; in lockstatus() local
1344 count = lkp->lk_count; in lockstatus()
1347 if (count & (LKC_XMASK | LKC_SMASK | LKC_EXREQ | LKC_UPREQ)) { in lockstatus()
1348 if (count & LKC_XMASK) { in lockstatus()
1353 } else if ((count & LKC_SMASK) && (count & LKC_SHARED)) { in lockstatus()
1368 uint64_t count; in lockowned() local
1370 count = lkp->lk_count; in lockowned()
1373 if (count & LKC_XMASK) in lockowned()
1376 return((count & LKC_SMASK) != 0); in lockowned()
1409 uint64_t count; in lockmgr_printinfo() local
1411 count = lkp->lk_count; in lockmgr_printinfo()
1419 if (count & LKC_XMASK) { in lockmgr_printinfo()
1421 lkp->lk_wmesg, (intmax_t)count, td, in lockmgr_printinfo()
1423 } else if ((count & LKC_SMASK) && (count & LKC_SHARED)) { in lockmgr_printinfo()
1425 lkp->lk_wmesg, (intmax_t)count); in lockmgr_printinfo()
1429 if ((count & (LKC_EXREQ | LKC_UPREQ)) || in lockmgr_printinfo()
1430 ((count & LKC_XMASK) && (count & LKC_SMASK))) in lockmgr_printinfo()