84 rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n)  in __weak_alias()
87 	return (uintptr_t)atomic_cas_ptr(&ptr->ptr_owner, (void *)o,  in __weak_alias()
92 pthread_rwlock_init(pthread_rwlock_t *ptr,  in pthread_rwlock_init()  argument
96 		return __libc_rwlock_init_stub(ptr, attr);  in pthread_rwlock_init()
101 	ptr->ptr_magic = _PT_RWLOCK_MAGIC;  in pthread_rwlock_init()
102 	PTQ_INIT(&ptr->ptr_rblocked);  in pthread_rwlock_init()
103 	PTQ_INIT(&ptr->ptr_wblocked);  in pthread_rwlock_init()
104 	ptr->ptr_nreaders = 0;  in pthread_rwlock_init()
105 	ptr->ptr_owner = NULL;  in pthread_rwlock_init()
112 pthread_rwlock_destroy(pthread_rwlock_t *ptr)  in pthread_rwlock_destroy()  argument
115 		return __libc_rwlock_destroy_stub(ptr);  in pthread_rwlock_destroy()
118 	    ptr->ptr_magic == _PT_RWLOCK_MAGIC);  in pthread_rwlock_destroy()
120 	if ((!PTQ_EMPTY(&ptr->ptr_rblocked)) ||  in pthread_rwlock_destroy()
121 	    (!PTQ_EMPTY(&ptr->ptr_wblocked)) ||  in pthread_rwlock_destroy()
122 	    (ptr->ptr_nreaders != 0) ||  in pthread_rwlock_destroy()
123 	    (ptr->ptr_owner != NULL))  in pthread_rwlock_destroy()
125 	ptr->ptr_magic = _PT_RWLOCK_DEAD;  in pthread_rwlock_destroy()
158 pthread__rwlock_rdlock(pthread_rwlock_t *ptr, const struct timespec *ts)  in pthread__rwlock_rdlock()  argument
166 	    ptr->ptr_magic == _PT_RWLOCK_MAGIC);  in pthread__rwlock_rdlock()
168 	for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {  in pthread__rwlock_rdlock()
174 			next = rw_cas(ptr, owner, owner + RW_READ_INCR);  in pthread__rwlock_rdlock()
197 				owner = (uintptr_t)ptr->ptr_owner;  in pthread__rwlock_rdlock()
207 		interlock = pthread__hashlock(ptr);  in pthread__rwlock_rdlock()
214 		next = rw_cas(ptr, owner, owner | RW_HAS_WAITERS);  in pthread__rwlock_rdlock()
221 	    	PTQ_INSERT_HEAD(&ptr->ptr_rblocked, self, pt_sleep);  in pthread__rwlock_rdlock()
222 	    	ptr->ptr_nreaders++;  in pthread__rwlock_rdlock()
224 		self->pt_sleepobj = &ptr->ptr_rblocked;  in pthread__rwlock_rdlock()
225 		error = pthread__park(self, interlock, &ptr->ptr_rblocked,  in pthread__rwlock_rdlock()
229 			pthread__rwlock_early(self, ptr, interlock);  in pthread__rwlock_rdlock()
247 pthread_rwlock_tryrdlock(pthread_rwlock_t *ptr)  in pthread_rwlock_tryrdlock()  argument
252 		return __libc_rwlock_tryrdlock_stub(ptr);  in pthread_rwlock_tryrdlock()
255 	    ptr->ptr_magic == _PT_RWLOCK_MAGIC);  in pthread_rwlock_tryrdlock()
262 	for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {  in pthread_rwlock_tryrdlock()
265 		next = rw_cas(ptr, owner, owner + RW_READ_INCR);  in pthread_rwlock_tryrdlock()
277 pthread__rwlock_wrlock(pthread_rwlock_t *ptr, const struct timespec *ts)  in pthread__rwlock_wrlock()  argument
288 	    ptr->ptr_magic == _PT_RWLOCK_MAGIC);  in pthread__rwlock_wrlock()
290 	for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {  in pthread__rwlock_wrlock()
296 			next = rw_cas(ptr, owner,  in pthread__rwlock_wrlock()
319 				owner = (uintptr_t)ptr->ptr_owner;  in pthread__rwlock_wrlock()
329 		interlock = pthread__hashlock(ptr);  in pthread__rwlock_wrlock()
336 		next = rw_cas(ptr, owner,  in pthread__rwlock_wrlock()
344 	    	PTQ_INSERT_TAIL(&ptr->ptr_wblocked, self, pt_sleep);  in pthread__rwlock_wrlock()
346 		self->pt_sleepobj = &ptr->ptr_wblocked;  in pthread__rwlock_wrlock()
347 		error = pthread__park(self, interlock, &ptr->ptr_wblocked,  in pthread__rwlock_wrlock()
351 			pthread__rwlock_early(self, ptr, interlock);  in pthread__rwlock_wrlock()
368 pthread_rwlock_trywrlock(pthread_rwlock_t *ptr)  in pthread_rwlock_trywrlock()  argument
374 		return __libc_rwlock_trywrlock_stub(ptr);  in pthread_rwlock_trywrlock()
377 	    ptr->ptr_magic == _PT_RWLOCK_MAGIC);  in pthread_rwlock_trywrlock()
382 	for (owner = (uintptr_t)ptr->ptr_owner;; owner = next) {  in pthread_rwlock_trywrlock()
385 		next = rw_cas(ptr, owner, (uintptr_t)self | RW_WRITE_LOCKED);  in pthread_rwlock_trywrlock()
397 pthread_rwlock_rdlock(pthread_rwlock_t *ptr)  in pthread_rwlock_rdlock()  argument
400 		return __libc_rwlock_rdlock_stub(ptr);  in pthread_rwlock_rdlock()
402 	return pthread__rwlock_rdlock(ptr, NULL);  in pthread_rwlock_rdlock()
406 pthread_rwlock_timedrdlock(pthread_rwlock_t *ptr,  in pthread_rwlock_timedrdlock()  argument
416 	return pthread__rwlock_rdlock(ptr, abs_timeout);  in pthread_rwlock_timedrdlock()
420 pthread_rwlock_wrlock(pthread_rwlock_t *ptr)  in pthread_rwlock_wrlock()  argument
423 		return __libc_rwlock_wrlock_stub(ptr);  in pthread_rwlock_wrlock()
425 	return pthread__rwlock_wrlock(ptr, NULL);  in pthread_rwlock_wrlock()
429 pthread_rwlock_timedwrlock(pthread_rwlock_t *ptr,  in pthread_rwlock_timedwrlock()  argument
439 	return pthread__rwlock_wrlock(ptr, abs_timeout);  in pthread_rwlock_timedwrlock()
444 pthread_rwlock_unlock(pthread_rwlock_t *ptr)  in pthread_rwlock_unlock()  argument
451 		return __libc_rwlock_unlock_stub(ptr);  in pthread_rwlock_unlock()
454 	    ptr->ptr_magic == _PT_RWLOCK_MAGIC);  in pthread_rwlock_unlock()
465 	owner = (uintptr_t)ptr->ptr_owner;  in pthread_rwlock_unlock()
487 			next = rw_cas(ptr, owner, new);  in pthread_rwlock_unlock()
500 		interlock = pthread__hashlock(ptr);  in pthread_rwlock_unlock()
502 		owner = (uintptr_t)ptr->ptr_owner;  in pthread_rwlock_unlock()
514 		if ((thread = PTQ_FIRST(&ptr->ptr_wblocked)) != NULL) {  in pthread_rwlock_unlock()
520 			else if (ptr->ptr_nreaders != 0)  in pthread_rwlock_unlock()
527 			(void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new);  in pthread_rwlock_unlock()
534 			pthread__unpark(&ptr->ptr_wblocked, self,  in pthread_rwlock_unlock()
538 			PTQ_FOREACH(thread, &ptr->ptr_rblocked, pt_sleep) {  in pthread_rwlock_unlock()
556 			(void)atomic_swap_ptr(&ptr->ptr_owner, (void *)new);  in pthread_rwlock_unlock()
559 			ptr->ptr_nreaders = 0;  in pthread_rwlock_unlock()
560 			pthread__unpark_all(&ptr->ptr_rblocked, self,  in pthread_rwlock_unlock()
575 pthread__rwlock_early(pthread_t self, pthread_rwlock_t *ptr,  in pthread__rwlock_early()  argument
588 	owner = (uintptr_t)ptr->ptr_owner;  in pthread__rwlock_early()
595 	if (!PTQ_EMPTY(&ptr->ptr_wblocked))  in pthread__rwlock_early()
597 	else if (ptr->ptr_nreaders != 0)  in pthread__rwlock_early()
604 		next = rw_cas(ptr, owner, newval);  in pthread__rwlock_early()
612 _pthread_rwlock_held_np(pthread_rwlock_t *ptr)  in _pthread_rwlock_held_np()  argument
614 	uintptr_t owner = (uintptr_t)ptr->ptr_owner;  in _pthread_rwlock_held_np()
622 _pthread_rwlock_rdheld_np(pthread_rwlock_t *ptr)  in _pthread_rwlock_rdheld_np()  argument
624 	uintptr_t owner = (uintptr_t)ptr->ptr_owner;  in _pthread_rwlock_rdheld_np()
630 _pthread_rwlock_wrheld_np(pthread_rwlock_t *ptr)  in _pthread_rwlock_wrheld_np()  argument
632 	uintptr_t owner = (uintptr_t)ptr->ptr_owner;  in _pthread_rwlock_wrheld_np()
645 	    ptr->ptra_magic == _PT_RWLOCKATTR_MAGIC);  in pthread_rwlockattr_getpshared()
656 	    ptr->ptra_magic == _PT_RWLOCKATTR_MAGIC);  in pthread_rwlockattr_setpshared()