Lines Matching defs:rw
82 .lc_name = "rw",
138 #define rw_wowner(rw) lv_rw_wowner(RW_READ_VALUE(rw))
144 #define rw_recursed(rw) ((rw)->rw_recurse != 0)
149 #define rw_wlocked(rw) (rw_wowner((rw)) == curthread)
156 #define rw_owner(rw) rw_wowner(rw)
172 struct rwlock *rw;
174 rw = (struct rwlock *)lock;
176 rw_rlock(rw);
178 rw_wlock(rw);
184 struct rwlock *rw;
186 rw = (struct rwlock *)lock;
188 return (rw_try_rlock(rw));
190 return (rw_try_wlock(rw));
196 struct rwlock *rw;
198 rw = (struct rwlock *)lock;
199 rw_assert(rw, RA_LOCKED | LA_NOTRECURSED);
200 if (rw->rw_lock & RW_LOCK_READ) {
201 rw_runlock(rw);
204 rw_wunlock(rw);
213 const struct rwlock *rw = (const struct rwlock *)lock;
214 uintptr_t x = rw->rw_lock;
216 *owner = rw_wowner(rw);
225 struct rwlock *rw;
228 rw = rwlock2rw(c);
232 ASSERT_ATOMIC_LOAD_PTR(rw->rw_lock,
234 &rw->rw_lock));
250 lock_init(&rw->lock_object, &lock_class_rw, name, NULL, flags);
251 rw->rw_lock = RW_UNLOCKED;
252 rw->rw_recurse = 0;
258 struct rwlock *rw;
260 rw = rwlock2rw(c);
262 KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock %p not unlocked", rw));
263 KASSERT(rw->rw_recurse == 0, ("rw lock %p still recursed", rw));
264 rw->rw_lock = RW_DESTROYED;
265 lock_destroy(&rw->lock_object);
288 struct rwlock *rw;
291 rw = rwlock2rw(c);
296 curthread, rw->lock_object.lo_name, file, line));
297 KASSERT(rw->rw_lock != RW_DESTROYED,
299 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
303 if (!_rw_write_lock_fetch(rw, &v, tid))
304 _rw_wlock_hard(rw, v, file, line);
306 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,
309 LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
310 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
315 __rw_try_wlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
329 curthread, rw->lock_object.lo_name, file, line));
330 KASSERT(rw->rw_lock != RW_DESTROYED,
337 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid))
341 if (v == tid && (rw->lock_object.lo_flags & LO_RECURSABLE)) {
342 rw->rw_recurse++;
343 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
350 LOCK_LOG_TRY("WLOCK", &rw->lock_object, 0, rval, file, line);
352 WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
356 rw, 0, 0, file, line, LOCKSTAT_WRITER);
365 struct rwlock *rw;
367 rw = rwlock2rw(c);
368 return (__rw_try_wlock_int(rw LOCK_FILE_LINE_ARG));
374 struct rwlock *rw;
376 rw = rwlock2rw(c);
378 KASSERT(rw->rw_lock != RW_DESTROYED,
381 WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
382 LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
386 _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line);
388 __rw_wunlock(rw, curthread, file, line);
414 __rw_rlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp
429 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, vp,
431 if (LOCK_LOG_TEST(&rw->lock_object, 0))
434 rw, (void *)*vp,
444 __rw_rlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
472 if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
475 all_time -= lockstat_nsecs(&rw->lock_object);
495 lock_profile_obtain_lock_failed(&rw->lock_object, false,
498 THREAD_CONTENDS_ON_LOCK(&rw->lock_object);
501 if (__rw_rlock_try(rw, td, &v, false LOCK_FILE_LINE_ARG))
516 if (LOCK_LOG_TEST(&rw->lock_object, 0))
519 __func__, rw, owner);
522 "lockname:\"%s\"", rw->lock_object.lo_name);
525 v = RW_READ_VALUE(rw);
536 v = RW_READ_VALUE(rw);
543 rw->lock_object.lo_name);
547 v = RW_READ_VALUE(rw);
573 ts = turnstile_trywait(&rw->lock_object);
579 v = RW_READ_VALUE(rw);
617 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v,
620 if (LOCK_LOG_TEST(&rw->lock_object, 0))
622 __func__, rw);
629 if (LOCK_LOG_TEST(&rw->lock_object, 0))
631 rw);
633 sleep_time -= lockstat_nsecs(&rw->lock_object);
635 MPASS(owner == rw_owner(rw));
638 sleep_time += lockstat_nsecs(&rw->lock_object);
641 if (LOCK_LOG_TEST(&rw->lock_object, 0))
643 __func__, rw);
644 v = RW_READ_VALUE(rw);
646 THREAD_CONTENTION_DONE(&rw->lock_object);
652 all_time += lockstat_nsecs(&rw->lock_object);
654 LOCKSTAT_RECORD4(rw__block, rw, sleep_time,
660 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
670 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
675 __rw_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
685 td, rw->lock_object.lo_name, file, line));
686 KASSERT(rw->rw_lock != RW_DESTROYED,
688 KASSERT(rw_wowner(rw) != td,
690 rw->lock_object.lo_name, file, line));
691 WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL);
693 v = RW_READ_VALUE(rw);
695 !__rw_rlock_try(rw, td, &v, true LOCK_FILE_LINE_ARG)))
696 __rw_rlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
698 lock_profile_obtain_lock_success(&rw->lock_object, false, 0, 0,
701 LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
702 WITNESS_LOCK(&rw->lock_object, 0, file, line);
709 struct rwlock *rw;
711 rw = rwlock2rw(c);
712 __rw_rlock_int(rw LOCK_FILE_LINE_ARG);
716 __rw_try_rlock_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
725 curthread, rw->lock_object.lo_name, file, line));
727 x = rw->rw_lock;
729 KASSERT(rw->rw_lock != RW_DESTROYED,
733 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &x, x + RW_ONE_READER)) {
734 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 1, file,
736 WITNESS_LOCK(&rw->lock_object, LOP_TRYLOCK, file, line);
738 rw, 0, 0, file, line, LOCKSTAT_READER);
745 LOCK_LOG_TRY("RLOCK", &rw->lock_object, 0, 0, file, line);
752 struct rwlock *rw;
754 rw = rwlock2rw(c);
755 return (__rw_try_rlock_int(rw LOCK_FILE_LINE_ARG));
759 __rw_runlock_try(struct rwlock *rw, struct thread *td, uintptr_t *vp)
764 if (atomic_fcmpset_rel_ptr(&rw->rw_lock, vp,
766 if (LOCK_LOG_TEST(&rw->lock_object, 0))
769 __func__, rw, (void *)*vp,
782 __rw_runlock_hard(struct rwlock *rw, struct thread *td, uintptr_t v
792 if (__rw_runlock_try(rw, td, &v))
799 turnstile_chain_lock(&rw->lock_object);
800 v = RW_READ_VALUE(rw);
802 if (__rw_runlock_try(rw, td, &v))
830 if (!atomic_fcmpset_rel_ptr(&rw->rw_lock, &v, setv))
832 if (LOCK_LOG_TEST(&rw->lock_object, 0))
834 __func__, rw);
843 ts = turnstile_lookup(&rw->lock_object);
846 rw, (void *)passedv, (void *)v);
853 turnstile_chain_unlock(&rw->lock_object);
855 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_READER);
859 _rw_runlock_cookie_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
864 KASSERT(rw->rw_lock != RW_DESTROYED,
866 __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
867 WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
868 LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
871 v = RW_READ_VALUE(rw);
874 !__rw_runlock_try(rw, td, &v)))
875 __rw_runlock_hard(rw, td, v LOCK_FILE_LINE_ARG);
877 lock_profile_release_lock(&rw->lock_object, false);
885 struct rwlock *rw;
887 rw = rwlock2rw(c);
888 _rw_runlock_cookie_int(rw LOCK_FILE_LINE_ARG);
917 struct rwlock *rw;
946 rw = rwlock2rw(c);
951 if (_rw_write_lock_fetch(rw, &v, tid))
956 all_time -= lockstat_nsecs(&rw->lock_object);
969 v = RW_READ_VALUE(rw);
972 KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE,
973 ("%s: recursing but non-recursive rw %s @ %s:%d\n",
974 __func__, rw->lock_object.lo_name, file, line));
975 rw->rw_recurse++;
976 atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
977 if (LOCK_LOG_TEST(&rw->lock_object, 0))
978 CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
982 if (LOCK_LOG_TEST(&rw->lock_object, 0))
984 rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
995 lock_profile_obtain_lock_failed(&rw->lock_object, false,
998 THREAD_CONTENDS_ON_LOCK(&rw->lock_object);
1002 if (_rw_write_lock_fetch(rw, &v, tid))
1012 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid))
1028 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1030 __func__, rw, owner);
1033 rw->lock_object.lo_name);
1036 v = RW_READ_VALUE(rw);
1052 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v,
1063 rw->lock_object.lo_name);
1067 v = RW_READ_VALUE(rw);
1086 ts = turnstile_trywait(&rw->lock_object);
1087 v = RW_READ_VALUE(rw);
1121 if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid | setv)) {
1137 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v, setv))
1151 if (!atomic_fcmpset_ptr(&rw->rw_lock, &v,
1154 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1156 __func__, rw);
1165 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1167 rw);
1169 sleep_time -= lockstat_nsecs(&rw->lock_object);
1171 MPASS(owner == rw_owner(rw));
1174 sleep_time += lockstat_nsecs(&rw->lock_object);
1177 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1179 __func__, rw);
1183 v = RW_READ_VALUE(rw);
1185 THREAD_CONTENTION_DONE(&rw->lock_object);
1197 all_time += lockstat_nsecs(&rw->lock_object);
1199 LOCKSTAT_RECORD4(rw__block, rw, sleep_time,
1205 LOCKSTAT_RECORD4(rw__spin, rw, all_time - sleep_time,
1210 LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, contested,
1223 struct rwlock *rw;
1232 rw = rwlock2rw(c);
1234 v = RW_READ_VALUE(rw);
1237 if (--(rw->rw_recurse) == 0)
1238 atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED);
1239 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1240 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
1244 LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, LOCKSTAT_WRITER);
1245 if (v == tid && _rw_write_unlock(rw, tid))
1248 KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
1251 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1252 CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
1254 turnstile_chain_lock(&rw->lock_object);
1274 v = RW_READ_VALUE(rw);
1280 atomic_store_rel_ptr(&rw->rw_lock, setv);
1283 if (LOCK_LOG_TEST(&rw->lock_object, 0))
1284 CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
1287 ts = turnstile_lookup(&rw->lock_object);
1289 panic("got NULL turnstile on rwlock %p passedv %p v %p", rw,
1294 turnstile_chain_unlock(&rw->lock_object);
1303 __rw_try_upgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
1312 KASSERT(rw->rw_lock != RW_DESTROYED,
1314 __rw_assert(&rw->rw_lock, RA_RLOCKED, file, line);
1325 v = RW_READ_VALUE(rw);
1330 success = atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, tid);
1339 ts = turnstile_trywait(&rw->lock_object);
1340 v = RW_READ_VALUE(rw);
1353 success = atomic_fcmpset_ptr(&rw->rw_lock, &v, setv);
1363 LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
1366 WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
1368 LOCKSTAT_RECORD0(rw__upgrade, rw);
1376 struct rwlock *rw;
1378 rw = rwlock2rw(c);
1379 return (__rw_try_upgrade_int(rw LOCK_FILE_LINE_ARG));
1386 __rw_downgrade_int(struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
1395 KASSERT(rw->rw_lock != RW_DESTROYED,
1397 __rw_assert(&rw->rw_lock, RA_WLOCKED | RA_NOTRECURSED, file, line);
1399 if (rw_recursed(rw))
1403 WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
1411 if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
1418 turnstile_chain_lock(&rw->lock_object);
1419 v = rw->rw_lock & RW_LOCK_WAITERS;
1428 ts = turnstile_lookup(&rw->lock_object);
1432 atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v);
1442 turnstile_chain_unlock(&rw->lock_object);
1445 LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
1446 LOCKSTAT_RECORD0(rw__downgrade, rw);
1452 struct rwlock *rw;
1454 rw = rwlock2rw(c);
1455 __rw_downgrade_int(rw LOCK_FILE_LINE_ARG);
1471 const struct rwlock *rw;
1476 rw = rwlock2rw(c);
1486 witness_assert(&rw->lock_object, what, file, line);
1493 if (rw->rw_lock == RW_UNLOCKED ||
1494 (!(rw->rw_lock & RW_LOCK_READ) && (what & RA_RLOCKED ||
1495 rw_wowner(rw) != curthread)))
1497 rw->lock_object.lo_name, (what & RA_RLOCKED) ?
1500 if (!(rw->rw_lock & RW_LOCK_READ) && !(what & RA_RLOCKED)) {
1501 if (rw_recursed(rw)) {
1504 rw->lock_object.lo_name, file,
1508 rw->lock_object.lo_name, file, line);
1515 if (rw_wowner(rw) != curthread)
1517 rw->lock_object.lo_name, file, line);
1518 if (rw_recursed(rw)) {
1521 rw->lock_object.lo_name, file, line);
1524 rw->lock_object.lo_name, file, line);
1528 witness_assert(&rw->lock_object, what, file, line);
1534 if (rw_wowner(rw) == curthread)
1536 rw->lock_object.lo_name, file, line);
1540 panic("Unknown rw lock assertion: %d @ %s:%d", what, file,
1550 const struct rwlock *rw;
1553 rw = (const struct rwlock *)lock;
1556 if (rw->rw_lock == RW_UNLOCKED)
1558 else if (rw->rw_lock == RW_DESTROYED) {
1561 } else if (rw->rw_lock & RW_LOCK_READ)
1563 (uintmax_t)(RW_READERS(rw->rw_lock)));
1565 td = rw_wowner(rw);
1568 if (rw_recursed(rw))
1569 db_printf(" recursed: %u\n", rw->rw_recurse);
1572 switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {