xref: /onnv-gate/usr/src/lib/libc/port/threads/synch.c (revision 10887:25ca697b9a84)
10Sstevel@tonic-gate /*
20Sstevel@tonic-gate  * CDDL HEADER START
30Sstevel@tonic-gate  *
40Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
51893Sraf  * Common Development and Distribution License (the "License").
61893Sraf  * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate  *
80Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate  * See the License for the specific language governing permissions
110Sstevel@tonic-gate  * and limitations under the License.
120Sstevel@tonic-gate  *
130Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate  *
190Sstevel@tonic-gate  * CDDL HEADER END
200Sstevel@tonic-gate  */
211219Sraf 
220Sstevel@tonic-gate /*
239170SRoger.Faulkner@Sun.COM  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
240Sstevel@tonic-gate  * Use is subject to license terms.
250Sstevel@tonic-gate  */
260Sstevel@tonic-gate 
270Sstevel@tonic-gate #include "lint.h"
280Sstevel@tonic-gate #include "thr_uberdata.h"
296247Sraf #include <sys/rtpriocntl.h>
306057Sraf #include <sys/sdt.h>
316057Sraf #include <atomic.h>
320Sstevel@tonic-gate 
336247Sraf #if defined(THREAD_DEBUG)
346247Sraf #define	INCR32(x)	(((x) != UINT32_MAX)? (x)++ : 0)
356247Sraf #define	INCR(x)		((x)++)
366247Sraf #define	DECR(x)		((x)--)
376247Sraf #define	MAXINCR(m, x)	((m < ++x)? (m = x) : 0)
386247Sraf #else
396247Sraf #define	INCR32(x)
406247Sraf #define	INCR(x)
416247Sraf #define	DECR(x)
426247Sraf #define	MAXINCR(m, x)
436247Sraf #endif
446247Sraf 
450Sstevel@tonic-gate /*
460Sstevel@tonic-gate  * This mutex is initialized to be held by lwp#1.
470Sstevel@tonic-gate  * It is used to block a thread that has returned from a mutex_lock()
484574Sraf  * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error.
490Sstevel@tonic-gate  */
500Sstevel@tonic-gate mutex_t	stall_mutex = DEFAULTMUTEX;
510Sstevel@tonic-gate 
520Sstevel@tonic-gate static int shared_mutex_held(mutex_t *);
534574Sraf static int mutex_queuelock_adaptive(mutex_t *);
544574Sraf static void mutex_wakeup_all(mutex_t *);
550Sstevel@tonic-gate 
560Sstevel@tonic-gate /*
570Sstevel@tonic-gate  * Lock statistics support functions.
580Sstevel@tonic-gate  */
590Sstevel@tonic-gate void
record_begin_hold(tdb_mutex_stats_t * msp)600Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp)
610Sstevel@tonic-gate {
620Sstevel@tonic-gate 	tdb_incr(msp->mutex_lock);
630Sstevel@tonic-gate 	msp->mutex_begin_hold = gethrtime();
640Sstevel@tonic-gate }
650Sstevel@tonic-gate 
660Sstevel@tonic-gate hrtime_t
record_hold_time(tdb_mutex_stats_t * msp)670Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp)
680Sstevel@tonic-gate {
690Sstevel@tonic-gate 	hrtime_t now = gethrtime();
700Sstevel@tonic-gate 
710Sstevel@tonic-gate 	if (msp->mutex_begin_hold)
720Sstevel@tonic-gate 		msp->mutex_hold_time += now - msp->mutex_begin_hold;
730Sstevel@tonic-gate 	msp->mutex_begin_hold = 0;
740Sstevel@tonic-gate 	return (now);
750Sstevel@tonic-gate }
760Sstevel@tonic-gate 
770Sstevel@tonic-gate /*
780Sstevel@tonic-gate  * Called once at library initialization.
790Sstevel@tonic-gate  */
800Sstevel@tonic-gate void
mutex_setup(void)810Sstevel@tonic-gate mutex_setup(void)
820Sstevel@tonic-gate {
830Sstevel@tonic-gate 	if (set_lock_byte(&stall_mutex.mutex_lockw))
840Sstevel@tonic-gate 		thr_panic("mutex_setup() cannot acquire stall_mutex");
850Sstevel@tonic-gate 	stall_mutex.mutex_owner = (uintptr_t)curthread;
860Sstevel@tonic-gate }
870Sstevel@tonic-gate 
880Sstevel@tonic-gate /*
895629Sraf  * The default spin count of 1000 is experimentally determined.
905629Sraf  * On sun4u machines with any number of processors it could be raised
910Sstevel@tonic-gate  * to 10,000 but that (experimentally) makes almost no difference.
925629Sraf  * The environment variable:
930Sstevel@tonic-gate  *	_THREAD_ADAPTIVE_SPIN=count
945629Sraf  * can be used to override and set the count in the range [0 .. 1,000,000].
950Sstevel@tonic-gate  */
960Sstevel@tonic-gate int	thread_adaptive_spin = 1000;
970Sstevel@tonic-gate uint_t	thread_max_spinners = 100;
980Sstevel@tonic-gate int	thread_queue_verify = 0;
990Sstevel@tonic-gate static	int	ncpus;
1000Sstevel@tonic-gate 
1010Sstevel@tonic-gate /*
1020Sstevel@tonic-gate  * Distinguish spinning for queue locks from spinning for regular locks.
1035629Sraf  * We try harder to acquire queue locks by spinning.
1040Sstevel@tonic-gate  * The environment variable:
1050Sstevel@tonic-gate  *	_THREAD_QUEUE_SPIN=count
1060Sstevel@tonic-gate  * can be used to override and set the count in the range [0 .. 1,000,000].
1070Sstevel@tonic-gate  */
1085629Sraf int	thread_queue_spin = 10000;
1090Sstevel@tonic-gate 
1104574Sraf #define	ALL_ATTRIBUTES				\
1114574Sraf 	(LOCK_RECURSIVE | LOCK_ERRORCHECK |	\
1124574Sraf 	LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT |	\
1134574Sraf 	LOCK_ROBUST)
1140Sstevel@tonic-gate 
1150Sstevel@tonic-gate /*
1164574Sraf  * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST,
1174574Sraf  * augmented by zero or more the flags:
1184574Sraf  *	LOCK_RECURSIVE
1194574Sraf  *	LOCK_ERRORCHECK
1204574Sraf  *	LOCK_PRIO_INHERIT
1214574Sraf  *	LOCK_PRIO_PROTECT
1224574Sraf  *	LOCK_ROBUST
1230Sstevel@tonic-gate  */
1246812Sraf #pragma weak _mutex_init = mutex_init
1250Sstevel@tonic-gate /* ARGSUSED2 */
1260Sstevel@tonic-gate int
mutex_init(mutex_t * mp,int type,void * arg)1276812Sraf mutex_init(mutex_t *mp, int type, void *arg)
1280Sstevel@tonic-gate {
1294574Sraf 	int basetype = (type & ~ALL_ATTRIBUTES);
1306247Sraf 	const pcclass_t *pccp;
1314574Sraf 	int error = 0;
1326247Sraf 	int ceil;
1334574Sraf 
1344574Sraf 	if (basetype == USYNC_PROCESS_ROBUST) {
1354574Sraf 		/*
1364574Sraf 		 * USYNC_PROCESS_ROBUST is a deprecated historical type.
1374574Sraf 		 * We change it into (USYNC_PROCESS | LOCK_ROBUST) but
1384574Sraf 		 * retain the USYNC_PROCESS_ROBUST flag so we can return
1394574Sraf 		 * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST
1404574Sraf 		 * mutexes will ever draw ELOCKUNMAPPED).
1414574Sraf 		 */
1424574Sraf 		type |= (USYNC_PROCESS | LOCK_ROBUST);
1434574Sraf 		basetype = USYNC_PROCESS;
1444574Sraf 	}
1454574Sraf 
1466247Sraf 	if (type & LOCK_PRIO_PROTECT)
1476247Sraf 		pccp = get_info_by_policy(SCHED_FIFO);
1486247Sraf 	if ((basetype != USYNC_THREAD && basetype != USYNC_PROCESS) ||
1494574Sraf 	    (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT))
1506247Sraf 	    == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT) ||
1516247Sraf 	    ((type & LOCK_PRIO_PROTECT) &&
1526247Sraf 	    ((ceil = *(int *)arg) < pccp->pcc_primin ||
1536247Sraf 	    ceil > pccp->pcc_primax))) {
1544574Sraf 		error = EINVAL;
1554574Sraf 	} else if (type & LOCK_ROBUST) {
1564574Sraf 		/*
1574574Sraf 		 * Callers of mutex_init() with the LOCK_ROBUST attribute
1584574Sraf 		 * are required to pass an initially all-zero mutex.
1594574Sraf 		 * Multiple calls to mutex_init() are allowed; all but
1604574Sraf 		 * the first return EBUSY.  A call to mutex_init() is
1614574Sraf 		 * allowed to make an inconsistent robust lock consistent
1624574Sraf 		 * (for historical usage, even though the proper interface
1634574Sraf 		 * for this is mutex_consistent()).  Note that we use
1644574Sraf 		 * atomic_or_16() to set the LOCK_INITED flag so as
1654574Sraf 		 * not to disturb surrounding bits (LOCK_OWNERDEAD, etc).
1664574Sraf 		 */
1674574Sraf 		if (!(mp->mutex_flag & LOCK_INITED)) {
1684574Sraf 			mp->mutex_type = (uint8_t)type;
1696812Sraf 			atomic_or_16(&mp->mutex_flag, LOCK_INITED);
1704574Sraf 			mp->mutex_magic = MUTEX_MAGIC;
1714574Sraf 		} else if (type != mp->mutex_type ||
1726247Sraf 		    ((type & LOCK_PRIO_PROTECT) && mp->mutex_ceiling != ceil)) {
1734574Sraf 			error = EINVAL;
1746812Sraf 		} else if (mutex_consistent(mp) != 0) {
1754574Sraf 			error = EBUSY;
1764574Sraf 		}
1774574Sraf 		/* register a process robust mutex with the kernel */
1784574Sraf 		if (basetype == USYNC_PROCESS)
1794574Sraf 			register_lock(mp);
1804574Sraf 	} else {
1816515Sraf 		(void) memset(mp, 0, sizeof (*mp));
1820Sstevel@tonic-gate 		mp->mutex_type = (uint8_t)type;
1830Sstevel@tonic-gate 		mp->mutex_flag = LOCK_INITED;
1844574Sraf 		mp->mutex_magic = MUTEX_MAGIC;
1850Sstevel@tonic-gate 	}
1864574Sraf 
1876247Sraf 	if (error == 0 && (type & LOCK_PRIO_PROTECT)) {
1886247Sraf 		mp->mutex_ceiling = ceil;
1896247Sraf 	}
1904574Sraf 
1917255Sraf 	/*
1927255Sraf 	 * This should be at the beginning of the function,
1937255Sraf 	 * but for the sake of old broken applications that
1947255Sraf 	 * do not have proper alignment for their mutexes
1957255Sraf 	 * (and don't check the return code from mutex_init),
1967255Sraf 	 * we put it here, after initializing the mutex regardless.
1977255Sraf 	 */
1987255Sraf 	if (error == 0 &&
1997255Sraf 	    ((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
2007255Sraf 	    curthread->ul_misaligned == 0)
2017255Sraf 		error = EINVAL;
2027255Sraf 
2030Sstevel@tonic-gate 	return (error);
2040Sstevel@tonic-gate }
2050Sstevel@tonic-gate 
2060Sstevel@tonic-gate /*
2076247Sraf  * Delete mp from list of ceiling mutexes owned by curthread.
2080Sstevel@tonic-gate  * Return 1 if the head of the chain was updated.
2090Sstevel@tonic-gate  */
2100Sstevel@tonic-gate int
_ceil_mylist_del(mutex_t * mp)2110Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp)
2120Sstevel@tonic-gate {
2130Sstevel@tonic-gate 	ulwp_t *self = curthread;
2140Sstevel@tonic-gate 	mxchain_t **mcpp;
2150Sstevel@tonic-gate 	mxchain_t *mcp;
2160Sstevel@tonic-gate 
2176247Sraf 	for (mcpp = &self->ul_mxchain;
2186247Sraf 	    (mcp = *mcpp) != NULL;
2196247Sraf 	    mcpp = &mcp->mxchain_next) {
2206247Sraf 		if (mcp->mxchain_mx == mp) {
2216247Sraf 			*mcpp = mcp->mxchain_next;
2226247Sraf 			lfree(mcp, sizeof (*mcp));
2236247Sraf 			return (mcpp == &self->ul_mxchain);
2246247Sraf 		}
2256247Sraf 	}
2266247Sraf 	return (0);
2270Sstevel@tonic-gate }
2280Sstevel@tonic-gate 
2290Sstevel@tonic-gate /*
2306247Sraf  * Add mp to the list of ceiling mutexes owned by curthread.
2310Sstevel@tonic-gate  * Return ENOMEM if no memory could be allocated.
2320Sstevel@tonic-gate  */
2330Sstevel@tonic-gate int
_ceil_mylist_add(mutex_t * mp)2340Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp)
2350Sstevel@tonic-gate {
2360Sstevel@tonic-gate 	ulwp_t *self = curthread;
2370Sstevel@tonic-gate 	mxchain_t *mcp;
2380Sstevel@tonic-gate 
2390Sstevel@tonic-gate 	if ((mcp = lmalloc(sizeof (*mcp))) == NULL)
2400Sstevel@tonic-gate 		return (ENOMEM);
2410Sstevel@tonic-gate 	mcp->mxchain_mx = mp;
2420Sstevel@tonic-gate 	mcp->mxchain_next = self->ul_mxchain;
2430Sstevel@tonic-gate 	self->ul_mxchain = mcp;
2440Sstevel@tonic-gate 	return (0);
2450Sstevel@tonic-gate }
2460Sstevel@tonic-gate 
2470Sstevel@tonic-gate /*
2486247Sraf  * Helper function for _ceil_prio_inherit() and _ceil_prio_waive(), below.
2496247Sraf  */
2506247Sraf static void
set_rt_priority(ulwp_t * self,int prio)2516247Sraf set_rt_priority(ulwp_t *self, int prio)
2526247Sraf {
2536247Sraf 	pcparms_t pcparm;
2546247Sraf 
2556247Sraf 	pcparm.pc_cid = self->ul_rtclassid;
2566247Sraf 	((rtparms_t *)pcparm.pc_clparms)->rt_tqnsecs = RT_NOCHANGE;
2576247Sraf 	((rtparms_t *)pcparm.pc_clparms)->rt_pri = prio;
2586515Sraf 	(void) priocntl(P_LWPID, self->ul_lwpid, PC_SETPARMS, &pcparm);
2596247Sraf }
2606247Sraf 
2616247Sraf /*
2626247Sraf  * Inherit priority from ceiling.
2636247Sraf  * This changes the effective priority, not the assigned priority.
2640Sstevel@tonic-gate  */
2650Sstevel@tonic-gate void
_ceil_prio_inherit(int prio)2666247Sraf _ceil_prio_inherit(int prio)
2670Sstevel@tonic-gate {
2680Sstevel@tonic-gate 	ulwp_t *self = curthread;
2696247Sraf 
2706247Sraf 	self->ul_epri = prio;
2716247Sraf 	set_rt_priority(self, prio);
2720Sstevel@tonic-gate }
2730Sstevel@tonic-gate 
2740Sstevel@tonic-gate /*
2750Sstevel@tonic-gate  * Waive inherited ceiling priority.  Inherit from head of owned ceiling locks
2760Sstevel@tonic-gate  * if holding at least one ceiling lock.  If no ceiling locks are held at this
2770Sstevel@tonic-gate  * point, disinherit completely, reverting back to assigned priority.
2780Sstevel@tonic-gate  */
2790Sstevel@tonic-gate void
_ceil_prio_waive(void)2800Sstevel@tonic-gate _ceil_prio_waive(void)
2810Sstevel@tonic-gate {
2820Sstevel@tonic-gate 	ulwp_t *self = curthread;
2836247Sraf 	mxchain_t *mcp = self->ul_mxchain;
2846247Sraf 	int prio;
2856247Sraf 
2866247Sraf 	if (mcp == NULL) {
2876247Sraf 		prio = self->ul_pri;
2886247Sraf 		self->ul_epri = 0;
2890Sstevel@tonic-gate 	} else {
2906247Sraf 		prio = mcp->mxchain_mx->mutex_ceiling;
2916247Sraf 		self->ul_epri = prio;
2920Sstevel@tonic-gate 	}
2936247Sraf 	set_rt_priority(self, prio);
2940Sstevel@tonic-gate }
2950Sstevel@tonic-gate 
2960Sstevel@tonic-gate /*
2975629Sraf  * Clear the lock byte.  Retain the waiters byte and the spinners byte.
2985629Sraf  * Return the old value of the lock word.
2995629Sraf  */
3005629Sraf static uint32_t
clear_lockbyte(volatile uint32_t * lockword)3015629Sraf clear_lockbyte(volatile uint32_t *lockword)
3025629Sraf {
3035629Sraf 	uint32_t old;
3045629Sraf 	uint32_t new;
3055629Sraf 
3065629Sraf 	do {
3075629Sraf 		old = *lockword;
3085629Sraf 		new = old & ~LOCKMASK;
3095629Sraf 	} while (atomic_cas_32(lockword, old, new) != old);
3105629Sraf 
3115629Sraf 	return (old);
3125629Sraf }
3135629Sraf 
3145629Sraf /*
3156057Sraf  * Same as clear_lockbyte(), but operates on mutex_lockword64.
3166057Sraf  * The mutex_ownerpid field is cleared along with the lock byte.
3176057Sraf  */
3186057Sraf static uint64_t
clear_lockbyte64(volatile uint64_t * lockword64)3196057Sraf clear_lockbyte64(volatile uint64_t *lockword64)
3206057Sraf {
3216057Sraf 	uint64_t old;
3226057Sraf 	uint64_t new;
3236057Sraf 
3246057Sraf 	do {
3256057Sraf 		old = *lockword64;
3266057Sraf 		new = old & ~LOCKMASK64;
3276057Sraf 	} while (atomic_cas_64(lockword64, old, new) != old);
3286057Sraf 
3296057Sraf 	return (old);
3306057Sraf }
3316057Sraf 
3326057Sraf /*
3336057Sraf  * Similar to set_lock_byte(), which only tries to set the lock byte.
3347255Sraf  * Here, we attempt to set the lock byte AND the mutex_ownerpid, keeping
3357255Sraf  * the remaining bytes constant.  This atomic operation is required for the
3367255Sraf  * correctness of process-shared robust locks, otherwise there would be
3377255Sraf  * a window or vulnerability in which the lock byte had been set but the
3387255Sraf  * mutex_ownerpid had not yet been set.  If the process were to die in
3397255Sraf  * this window of vulnerability (due to some other thread calling exit()
3407255Sraf  * or the process receiving a fatal signal), the mutex would be left locked
3417255Sraf  * but without a process-ID to determine which process was holding the lock.
3427255Sraf  * The kernel would then be unable to mark the robust mutex as LOCK_OWNERDEAD
3437255Sraf  * when the process died.  For all other cases of process-shared locks, this
3447255Sraf  * operation is just a convenience, for the sake of common code.
3457255Sraf  *
3467255Sraf  * This operation requires process-shared robust locks to be properly
3477255Sraf  * aligned on an 8-byte boundary, at least on sparc machines, lest the
3487255Sraf  * operation incur an alignment fault.  This is automatic when locks
3497255Sraf  * are declared properly using the mutex_t or pthread_mutex_t data types
3507255Sraf  * and the application does not allocate dynamic memory on less than an
3517255Sraf  * 8-byte boundary.  See the 'horrible hack' comments below for cases
3527255Sraf  * dealing with such broken applications.
3536057Sraf  */
3546057Sraf static int
set_lock_byte64(volatile uint64_t * lockword64,pid_t ownerpid)3556057Sraf set_lock_byte64(volatile uint64_t *lockword64, pid_t ownerpid)
3566057Sraf {
3576057Sraf 	uint64_t old;
3586057Sraf 	uint64_t new;
3596057Sraf 
3606057Sraf 	old = *lockword64 & ~LOCKMASK64;
3616057Sraf 	new = old | ((uint64_t)(uint_t)ownerpid << PIDSHIFT) | LOCKBYTE64;
3626057Sraf 	if (atomic_cas_64(lockword64, old, new) == old)
3636057Sraf 		return (LOCKCLEAR);
3646057Sraf 
3656057Sraf 	return (LOCKSET);
3666057Sraf }
3676057Sraf 
3686057Sraf /*
3695629Sraf  * Increment the spinners count in the mutex lock word.
3705629Sraf  * Return 0 on success.  Return -1 if the count would overflow.
3715629Sraf  */
3725629Sraf static int
spinners_incr(volatile uint32_t * lockword,uint8_t max_spinners)3735629Sraf spinners_incr(volatile uint32_t *lockword, uint8_t max_spinners)
3745629Sraf {
3755629Sraf 	uint32_t old;
3765629Sraf 	uint32_t new;
3775629Sraf 
3785629Sraf 	do {
3795629Sraf 		old = *lockword;
3805629Sraf 		if (((old & SPINNERMASK) >> SPINNERSHIFT) >= max_spinners)
3815629Sraf 			return (-1);
3825629Sraf 		new = old + (1 << SPINNERSHIFT);
3835629Sraf 	} while (atomic_cas_32(lockword, old, new) != old);
3845629Sraf 
3855629Sraf 	return (0);
3865629Sraf }
3875629Sraf 
3885629Sraf /*
3895629Sraf  * Decrement the spinners count in the mutex lock word.
3905629Sraf  * Return the new value of the lock word.
3915629Sraf  */
3925629Sraf static uint32_t
spinners_decr(volatile uint32_t * lockword)3935629Sraf spinners_decr(volatile uint32_t *lockword)
3945629Sraf {
3955629Sraf 	uint32_t old;
3965629Sraf 	uint32_t new;
3975629Sraf 
3985629Sraf 	do {
3995629Sraf 		new = old = *lockword;
4005629Sraf 		if (new & SPINNERMASK)
4015629Sraf 			new -= (1 << SPINNERSHIFT);
4025629Sraf 	} while (atomic_cas_32(lockword, old, new) != old);
4035629Sraf 
4045629Sraf 	return (new);
4055629Sraf }
4065629Sraf 
4075629Sraf /*
4080Sstevel@tonic-gate  * Non-preemptive spin locks.  Used by queue_lock().
4090Sstevel@tonic-gate  * No lock statistics are gathered for these locks.
4105629Sraf  * No DTrace probes are provided for these locks.
4110Sstevel@tonic-gate  */
4120Sstevel@tonic-gate void
spin_lock_set(mutex_t * mp)4130Sstevel@tonic-gate spin_lock_set(mutex_t *mp)
4140Sstevel@tonic-gate {
4150Sstevel@tonic-gate 	ulwp_t *self = curthread;
4160Sstevel@tonic-gate 
4170Sstevel@tonic-gate 	no_preempt(self);
4180Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
4190Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
4200Sstevel@tonic-gate 		return;
4210Sstevel@tonic-gate 	}
4220Sstevel@tonic-gate 	/*
4230Sstevel@tonic-gate 	 * Spin for a while, attempting to acquire the lock.
4240Sstevel@tonic-gate 	 */
4256247Sraf 	INCR32(self->ul_spin_lock_spin);
4260Sstevel@tonic-gate 	if (mutex_queuelock_adaptive(mp) == 0 ||
4270Sstevel@tonic-gate 	    set_lock_byte(&mp->mutex_lockw) == 0) {
4280Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
4290Sstevel@tonic-gate 		return;
4300Sstevel@tonic-gate 	}
4310Sstevel@tonic-gate 	/*
4320Sstevel@tonic-gate 	 * Try harder if we were previously at a no premption level.
4330Sstevel@tonic-gate 	 */
4340Sstevel@tonic-gate 	if (self->ul_preempt > 1) {
4356247Sraf 		INCR32(self->ul_spin_lock_spin2);
4360Sstevel@tonic-gate 		if (mutex_queuelock_adaptive(mp) == 0 ||
4370Sstevel@tonic-gate 		    set_lock_byte(&mp->mutex_lockw) == 0) {
4380Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
4390Sstevel@tonic-gate 			return;
4400Sstevel@tonic-gate 		}
4410Sstevel@tonic-gate 	}
4420Sstevel@tonic-gate 	/*
4430Sstevel@tonic-gate 	 * Give up and block in the kernel for the mutex.
4440Sstevel@tonic-gate 	 */
4456247Sraf 	INCR32(self->ul_spin_lock_sleep);
446*10887SRoger.Faulkner@Sun.COM 	(void) ___lwp_mutex_timedlock(mp, NULL, self);
4470Sstevel@tonic-gate }
4480Sstevel@tonic-gate 
4490Sstevel@tonic-gate void
spin_lock_clear(mutex_t * mp)4500Sstevel@tonic-gate spin_lock_clear(mutex_t *mp)
4510Sstevel@tonic-gate {
4520Sstevel@tonic-gate 	ulwp_t *self = curthread;
4530Sstevel@tonic-gate 
4540Sstevel@tonic-gate 	mp->mutex_owner = 0;
4554570Sraf 	if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) {
4564574Sraf 		(void) ___lwp_mutex_wakeup(mp, 0);
4576247Sraf 		INCR32(self->ul_spin_lock_wakeup);
4580Sstevel@tonic-gate 	}
4590Sstevel@tonic-gate 	preempt(self);
4600Sstevel@tonic-gate }
4610Sstevel@tonic-gate 
4620Sstevel@tonic-gate /*
4630Sstevel@tonic-gate  * Allocate the sleep queue hash table.
4640Sstevel@tonic-gate  */
4650Sstevel@tonic-gate void
queue_alloc(void)4660Sstevel@tonic-gate queue_alloc(void)
4670Sstevel@tonic-gate {
4680Sstevel@tonic-gate 	ulwp_t *self = curthread;
4690Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
4706247Sraf 	queue_head_t *qp;
4710Sstevel@tonic-gate 	void *data;
4720Sstevel@tonic-gate 	int i;
4730Sstevel@tonic-gate 
4740Sstevel@tonic-gate 	/*
4750Sstevel@tonic-gate 	 * No locks are needed; we call here only when single-threaded.
4760Sstevel@tonic-gate 	 */
4770Sstevel@tonic-gate 	ASSERT(self == udp->ulwp_one);
4780Sstevel@tonic-gate 	ASSERT(!udp->uberflags.uf_mt);
4796515Sraf 	if ((data = mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t),
4800Sstevel@tonic-gate 	    PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0))
4810Sstevel@tonic-gate 	    == MAP_FAILED)
4820Sstevel@tonic-gate 		thr_panic("cannot allocate thread queue_head table");
4836247Sraf 	udp->queue_head = qp = (queue_head_t *)data;
4846247Sraf 	for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
4856247Sraf 		qp->qh_type = (i < QHASHSIZE)? MX : CV;
4866247Sraf 		qp->qh_lock.mutex_flag = LOCK_INITED;
4876247Sraf 		qp->qh_lock.mutex_magic = MUTEX_MAGIC;
4886247Sraf 		qp->qh_hlist = &qp->qh_def_root;
4896247Sraf #if defined(THREAD_DEBUG)
4906247Sraf 		qp->qh_hlen = 1;
4916247Sraf 		qp->qh_hmax = 1;
4926247Sraf #endif
4934574Sraf 	}
4940Sstevel@tonic-gate }
4950Sstevel@tonic-gate 
4960Sstevel@tonic-gate #if defined(THREAD_DEBUG)
4970Sstevel@tonic-gate 
4980Sstevel@tonic-gate /*
4990Sstevel@tonic-gate  * Debugging: verify correctness of a sleep queue.
5000Sstevel@tonic-gate  */
5010Sstevel@tonic-gate void
QVERIFY(queue_head_t * qp)5020Sstevel@tonic-gate QVERIFY(queue_head_t *qp)
5030Sstevel@tonic-gate {
5040Sstevel@tonic-gate 	ulwp_t *self = curthread;
5050Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
5066247Sraf 	queue_root_t *qrp;
5070Sstevel@tonic-gate 	ulwp_t *ulwp;
5080Sstevel@tonic-gate 	ulwp_t *prev;
5090Sstevel@tonic-gate 	uint_t index;
5106247Sraf 	uint32_t cnt;
5110Sstevel@tonic-gate 	char qtype;
5120Sstevel@tonic-gate 	void *wchan;
5130Sstevel@tonic-gate 
5140Sstevel@tonic-gate 	ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE);
5150Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
5166247Sraf 	for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) {
5176247Sraf 		cnt++;
5186247Sraf 		ASSERT((qrp->qr_head != NULL && qrp->qr_tail != NULL) ||
5196247Sraf 		    (qrp->qr_head == NULL && qrp->qr_tail == NULL));
5206247Sraf 	}
5216247Sraf 	ASSERT(qp->qh_hlen == cnt && qp->qh_hmax >= cnt);
5226247Sraf 	qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV;
5236247Sraf 	ASSERT(qp->qh_type == qtype);
5240Sstevel@tonic-gate 	if (!thread_queue_verify)
5250Sstevel@tonic-gate 		return;
5260Sstevel@tonic-gate 	/* real expensive stuff, only for _THREAD_QUEUE_VERIFY */
5276247Sraf 	for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) {
5286247Sraf 		for (prev = NULL, ulwp = qrp->qr_head; ulwp != NULL;
5296247Sraf 		    prev = ulwp, ulwp = ulwp->ul_link) {
5306247Sraf 			cnt++;
5316247Sraf 			if (ulwp->ul_writer)
5326247Sraf 				ASSERT(prev == NULL || prev->ul_writer);
5336247Sraf 			ASSERT(ulwp->ul_qtype == qtype);
5346247Sraf 			ASSERT(ulwp->ul_wchan != NULL);
5356247Sraf 			ASSERT(ulwp->ul_sleepq == qp);
5366247Sraf 			wchan = ulwp->ul_wchan;
5376247Sraf 			ASSERT(qrp->qr_wchan == wchan);
5386247Sraf 			index = QUEUE_HASH(wchan, qtype);
5396247Sraf 			ASSERT(&udp->queue_head[index] == qp);
5406247Sraf 		}
5416247Sraf 		ASSERT(qrp->qr_tail == prev);
5420Sstevel@tonic-gate 	}
5430Sstevel@tonic-gate 	ASSERT(qp->qh_qlen == cnt);
5440Sstevel@tonic-gate }
5450Sstevel@tonic-gate 
5460Sstevel@tonic-gate #else	/* THREAD_DEBUG */
5470Sstevel@tonic-gate 
5480Sstevel@tonic-gate #define	QVERIFY(qp)
5490Sstevel@tonic-gate 
5500Sstevel@tonic-gate #endif	/* THREAD_DEBUG */
5510Sstevel@tonic-gate 
5520Sstevel@tonic-gate /*
5530Sstevel@tonic-gate  * Acquire a queue head.
5540Sstevel@tonic-gate  */
5550Sstevel@tonic-gate queue_head_t *
queue_lock(void * wchan,int qtype)5560Sstevel@tonic-gate queue_lock(void *wchan, int qtype)
5570Sstevel@tonic-gate {
5580Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
5590Sstevel@tonic-gate 	queue_head_t *qp;
5606247Sraf 	queue_root_t *qrp;
5610Sstevel@tonic-gate 
5620Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
5630Sstevel@tonic-gate 
5640Sstevel@tonic-gate 	/*
5650Sstevel@tonic-gate 	 * It is possible that we could be called while still single-threaded.
5660Sstevel@tonic-gate 	 * If so, we call queue_alloc() to allocate the queue_head[] array.
5670Sstevel@tonic-gate 	 */
5680Sstevel@tonic-gate 	if ((qp = udp->queue_head) == NULL) {
5690Sstevel@tonic-gate 		queue_alloc();
5700Sstevel@tonic-gate 		qp = udp->queue_head;
5710Sstevel@tonic-gate 	}
5720Sstevel@tonic-gate 	qp += QUEUE_HASH(wchan, qtype);
5730Sstevel@tonic-gate 	spin_lock_set(&qp->qh_lock);
5746247Sraf 	for (qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next)
5756247Sraf 		if (qrp->qr_wchan == wchan)
5766247Sraf 			break;
5776247Sraf 	if (qrp == NULL && qp->qh_def_root.qr_head == NULL) {
5786247Sraf 		/* the default queue root is available; use it */
5796247Sraf 		qrp = &qp->qh_def_root;
5806247Sraf 		qrp->qr_wchan = wchan;
5816247Sraf 		ASSERT(qrp->qr_next == NULL);
5826247Sraf 		ASSERT(qrp->qr_tail == NULL &&
5836247Sraf 		    qrp->qr_rtcount == 0 && qrp->qr_qlen == 0);
5846247Sraf 	}
5856247Sraf 	qp->qh_wchan = wchan;	/* valid until queue_unlock() is called */
5866247Sraf 	qp->qh_root = qrp;	/* valid until queue_unlock() is called */
5876247Sraf 	INCR32(qp->qh_lockcount);
5880Sstevel@tonic-gate 	QVERIFY(qp);
5890Sstevel@tonic-gate 	return (qp);
5900Sstevel@tonic-gate }
5910Sstevel@tonic-gate 
5920Sstevel@tonic-gate /*
5930Sstevel@tonic-gate  * Release a queue head.
5940Sstevel@tonic-gate  */
5950Sstevel@tonic-gate void
queue_unlock(queue_head_t * qp)5960Sstevel@tonic-gate queue_unlock(queue_head_t *qp)
5970Sstevel@tonic-gate {
5980Sstevel@tonic-gate 	QVERIFY(qp);
5990Sstevel@tonic-gate 	spin_lock_clear(&qp->qh_lock);
6000Sstevel@tonic-gate }
6010Sstevel@tonic-gate 
6020Sstevel@tonic-gate /*
6030Sstevel@tonic-gate  * For rwlock queueing, we must queue writers ahead of readers of the
6040Sstevel@tonic-gate  * same priority.  We do this by making writers appear to have a half
6050Sstevel@tonic-gate  * point higher priority for purposes of priority comparisons below.
6060Sstevel@tonic-gate  */
6070Sstevel@tonic-gate #define	CMP_PRIO(ulwp)	((real_priority(ulwp) << 1) + (ulwp)->ul_writer)
6080Sstevel@tonic-gate 
6090Sstevel@tonic-gate void
enqueue(queue_head_t * qp,ulwp_t * ulwp,int force_fifo)6106247Sraf enqueue(queue_head_t *qp, ulwp_t *ulwp, int force_fifo)
6110Sstevel@tonic-gate {
6126247Sraf 	queue_root_t *qrp;
6130Sstevel@tonic-gate 	ulwp_t **ulwpp;
6140Sstevel@tonic-gate 	ulwp_t *next;
6150Sstevel@tonic-gate 	int pri = CMP_PRIO(ulwp);
6166247Sraf 
6170Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
6180Sstevel@tonic-gate 	ASSERT(ulwp->ul_sleepq != qp);
6190Sstevel@tonic-gate 
6206247Sraf 	if ((qrp = qp->qh_root) == NULL) {
6216247Sraf 		/* use the thread's queue root for the linkage */
6226247Sraf 		qrp = &ulwp->ul_queue_root;
6236247Sraf 		qrp->qr_next = qp->qh_hlist;
6246247Sraf 		qrp->qr_prev = NULL;
6256247Sraf 		qrp->qr_head = NULL;
6266247Sraf 		qrp->qr_tail = NULL;
6276247Sraf 		qrp->qr_wchan = qp->qh_wchan;
6286247Sraf 		qrp->qr_rtcount = 0;
6296247Sraf 		qrp->qr_qlen = 0;
6306247Sraf 		qrp->qr_qmax = 0;
6316247Sraf 		qp->qh_hlist->qr_prev = qrp;
6326247Sraf 		qp->qh_hlist = qrp;
6336247Sraf 		qp->qh_root = qrp;
6346247Sraf 		MAXINCR(qp->qh_hmax, qp->qh_hlen);
6356247Sraf 	}
6366247Sraf 
6370Sstevel@tonic-gate 	/*
6380Sstevel@tonic-gate 	 * LIFO queue ordering is unfair and can lead to starvation,
6390Sstevel@tonic-gate 	 * but it gives better performance for heavily contended locks.
6400Sstevel@tonic-gate 	 * We use thread_queue_fifo (range is 0..8) to determine
6410Sstevel@tonic-gate 	 * the frequency of FIFO vs LIFO queuing:
6420Sstevel@tonic-gate 	 *	0 : every 256th time	(almost always LIFO)
6430Sstevel@tonic-gate 	 *	1 : every 128th time
6440Sstevel@tonic-gate 	 *	2 : every 64th  time
6450Sstevel@tonic-gate 	 *	3 : every 32nd  time
6460Sstevel@tonic-gate 	 *	4 : every 16th  time	(the default value, mostly LIFO)
6470Sstevel@tonic-gate 	 *	5 : every 8th   time
6480Sstevel@tonic-gate 	 *	6 : every 4th   time
6490Sstevel@tonic-gate 	 *	7 : every 2nd   time
6500Sstevel@tonic-gate 	 *	8 : every time		(never LIFO, always FIFO)
6510Sstevel@tonic-gate 	 * Note that there is always some degree of FIFO ordering.
6520Sstevel@tonic-gate 	 * This breaks live lock conditions that occur in applications
6530Sstevel@tonic-gate 	 * that are written assuming (incorrectly) that threads acquire
6540Sstevel@tonic-gate 	 * locks fairly, that is, in roughly round-robin order.
6556247Sraf 	 * In any event, the queue is maintained in kernel priority order.
6560Sstevel@tonic-gate 	 *
6576247Sraf 	 * If force_fifo is non-zero, fifo queueing is forced.
6580Sstevel@tonic-gate 	 * SUSV3 requires this for semaphores.
6590Sstevel@tonic-gate 	 */
6606247Sraf 	if (qrp->qr_head == NULL) {
6610Sstevel@tonic-gate 		/*
6620Sstevel@tonic-gate 		 * The queue is empty.  LIFO/FIFO doesn't matter.
6630Sstevel@tonic-gate 		 */
6646247Sraf 		ASSERT(qrp->qr_tail == NULL);
6656247Sraf 		ulwpp = &qrp->qr_head;
6666247Sraf 	} else if (force_fifo |
6676247Sraf 	    (((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0)) {
6680Sstevel@tonic-gate 		/*
6690Sstevel@tonic-gate 		 * Enqueue after the last thread whose priority is greater
6700Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
6710Sstevel@tonic-gate 		 * Attempt first to go directly onto the tail of the queue.
6720Sstevel@tonic-gate 		 */
6736247Sraf 		if (pri <= CMP_PRIO(qrp->qr_tail))
6746247Sraf 			ulwpp = &qrp->qr_tail->ul_link;
6750Sstevel@tonic-gate 		else {
6766247Sraf 			for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL;
6770Sstevel@tonic-gate 			    ulwpp = &next->ul_link)
6780Sstevel@tonic-gate 				if (pri > CMP_PRIO(next))
6790Sstevel@tonic-gate 					break;
6800Sstevel@tonic-gate 		}
6810Sstevel@tonic-gate 	} else {
6820Sstevel@tonic-gate 		/*
6830Sstevel@tonic-gate 		 * Enqueue before the first thread whose priority is less
6840Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
6850Sstevel@tonic-gate 		 * Hopefully we can go directly onto the head of the queue.
6860Sstevel@tonic-gate 		 */
6876247Sraf 		for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL;
6880Sstevel@tonic-gate 		    ulwpp = &next->ul_link)
6890Sstevel@tonic-gate 			if (pri >= CMP_PRIO(next))
6900Sstevel@tonic-gate 				break;
6910Sstevel@tonic-gate 	}
6920Sstevel@tonic-gate 	if ((ulwp->ul_link = *ulwpp) == NULL)
6936247Sraf 		qrp->qr_tail = ulwp;
6940Sstevel@tonic-gate 	*ulwpp = ulwp;
6950Sstevel@tonic-gate 
6960Sstevel@tonic-gate 	ulwp->ul_sleepq = qp;
6976247Sraf 	ulwp->ul_wchan = qp->qh_wchan;
6986247Sraf 	ulwp->ul_qtype = qp->qh_type;
6996247Sraf 	if ((ulwp->ul_schedctl != NULL &&
7006247Sraf 	    ulwp->ul_schedctl->sc_cid == ulwp->ul_rtclassid) |
7016247Sraf 	    ulwp->ul_pilocks) {
7026247Sraf 		ulwp->ul_rtqueued = 1;
7036247Sraf 		qrp->qr_rtcount++;
7046247Sraf 	}
7056247Sraf 	MAXINCR(qrp->qr_qmax, qrp->qr_qlen);
7066247Sraf 	MAXINCR(qp->qh_qmax, qp->qh_qlen);
7076247Sraf }
7086247Sraf 
7096247Sraf /*
7106247Sraf  * Helper function for queue_slot() and queue_slot_rt().
7116247Sraf  * Try to find a non-suspended thread on the queue.
7126247Sraf  */
7136247Sraf static ulwp_t **
queue_slot_runnable(ulwp_t ** ulwpp,ulwp_t ** prevp,int rt)7146247Sraf queue_slot_runnable(ulwp_t **ulwpp, ulwp_t **prevp, int rt)
7156247Sraf {
7166247Sraf 	ulwp_t *ulwp;
7176247Sraf 	ulwp_t **foundpp = NULL;
7186247Sraf 	int priority = -1;
7196247Sraf 	ulwp_t *prev;
7206247Sraf 	int tpri;
7216247Sraf 
7226247Sraf 	for (prev = NULL;
7236247Sraf 	    (ulwp = *ulwpp) != NULL;
7246247Sraf 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
7256247Sraf 		if (ulwp->ul_stop)	/* skip suspended threads */
7266247Sraf 			continue;
7276247Sraf 		tpri = rt? CMP_PRIO(ulwp) : 0;
7286247Sraf 		if (tpri > priority) {
7296247Sraf 			foundpp = ulwpp;
7306247Sraf 			*prevp = prev;
7316247Sraf 			priority = tpri;
7326247Sraf 			if (!rt)
7336247Sraf 				break;
7346247Sraf 		}
7356247Sraf 	}
7366247Sraf 	return (foundpp);
7370Sstevel@tonic-gate }
7380Sstevel@tonic-gate 
7390Sstevel@tonic-gate /*
7406247Sraf  * For real-time, we search the entire queue because the dispatch
7416247Sraf  * (kernel) priorities may have changed since enqueueing.
7420Sstevel@tonic-gate  */
7430Sstevel@tonic-gate static ulwp_t **
queue_slot_rt(ulwp_t ** ulwpp_org,ulwp_t ** prevp)7446247Sraf queue_slot_rt(ulwp_t **ulwpp_org, ulwp_t **prevp)
7456247Sraf {
7466247Sraf 	ulwp_t **ulwpp = ulwpp_org;
7476247Sraf 	ulwp_t *ulwp = *ulwpp;
7486247Sraf 	ulwp_t **foundpp = ulwpp;
7496247Sraf 	int priority = CMP_PRIO(ulwp);
7506247Sraf 	ulwp_t *prev;
7516247Sraf 	int tpri;
7526247Sraf 
7536247Sraf 	for (prev = ulwp, ulwpp = &ulwp->ul_link;
7546247Sraf 	    (ulwp = *ulwpp) != NULL;
7556247Sraf 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
7566247Sraf 		tpri = CMP_PRIO(ulwp);
7576247Sraf 		if (tpri > priority) {
7586247Sraf 			foundpp = ulwpp;
7596247Sraf 			*prevp = prev;
7606247Sraf 			priority = tpri;
7616247Sraf 		}
7626247Sraf 	}
7636247Sraf 	ulwp = *foundpp;
7646247Sraf 
7656247Sraf 	/*
7666247Sraf 	 * Try not to return a suspended thread.
7676247Sraf 	 * This mimics the old libthread's behavior.
7686247Sraf 	 */
7696247Sraf 	if (ulwp->ul_stop &&
7706247Sraf 	    (ulwpp = queue_slot_runnable(ulwpp_org, prevp, 1)) != NULL) {
7716247Sraf 		foundpp = ulwpp;
7726247Sraf 		ulwp = *foundpp;
7736247Sraf 	}
7746247Sraf 	ulwp->ul_rt = 1;
7756247Sraf 	return (foundpp);
7766247Sraf }
7776247Sraf 
7786247Sraf ulwp_t **
queue_slot(queue_head_t * qp,ulwp_t ** prevp,int * more)7796247Sraf queue_slot(queue_head_t *qp, ulwp_t **prevp, int *more)
7806247Sraf {
7816247Sraf 	queue_root_t *qrp;
7826247Sraf 	ulwp_t **ulwpp;
7836247Sraf 	ulwp_t *ulwp;
7846247Sraf 	int rt;
7856247Sraf 
7866247Sraf 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
7876247Sraf 
7886247Sraf 	if ((qrp = qp->qh_root) == NULL || (ulwp = qrp->qr_head) == NULL) {
7896247Sraf 		*more = 0;
7906247Sraf 		return (NULL);		/* no lwps on the queue */
7916247Sraf 	}
7926247Sraf 	rt = (qrp->qr_rtcount != 0);
7936247Sraf 	*prevp = NULL;
7946247Sraf 	if (ulwp->ul_link == NULL) {	/* only one lwp on the queue */
7956247Sraf 		*more = 0;
7966247Sraf 		ulwp->ul_rt = rt;
7976247Sraf 		return (&qrp->qr_head);
7986247Sraf 	}
7996247Sraf 	*more = 1;
8006247Sraf 
8016247Sraf 	if (rt)		/* real-time queue */
8026247Sraf 		return (queue_slot_rt(&qrp->qr_head, prevp));
8036247Sraf 	/*
8046247Sraf 	 * Try not to return a suspended thread.
8056247Sraf 	 * This mimics the old libthread's behavior.
8066247Sraf 	 */
8076247Sraf 	if (ulwp->ul_stop &&
8086247Sraf 	    (ulwpp = queue_slot_runnable(&qrp->qr_head, prevp, 0)) != NULL) {
8096247Sraf 		ulwp = *ulwpp;
8106247Sraf 		ulwp->ul_rt = 0;
8116247Sraf 		return (ulwpp);
8126247Sraf 	}
8136247Sraf 	/*
8146247Sraf 	 * The common case; just pick the first thread on the queue.
8156247Sraf 	 */
8166247Sraf 	ulwp->ul_rt = 0;
8176247Sraf 	return (&qrp->qr_head);
8186247Sraf }
8196247Sraf 
8206247Sraf /*
8216247Sraf  * Common code for unlinking an lwp from a user-level sleep queue.
8226247Sraf  */
8236247Sraf void
queue_unlink(queue_head_t * qp,ulwp_t ** ulwpp,ulwp_t * prev)8246247Sraf queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev)
8256247Sraf {
8266247Sraf 	queue_root_t *qrp = qp->qh_root;
8276247Sraf 	queue_root_t *nqrp;
8286247Sraf 	ulwp_t *ulwp = *ulwpp;
8296247Sraf 	ulwp_t *next;
8306247Sraf 
8316247Sraf 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
8326247Sraf 	ASSERT(qp->qh_wchan != NULL && ulwp->ul_wchan == qp->qh_wchan);
8336247Sraf 
8346247Sraf 	DECR(qp->qh_qlen);
8356247Sraf 	DECR(qrp->qr_qlen);
8366247Sraf 	if (ulwp->ul_rtqueued) {
8376247Sraf 		ulwp->ul_rtqueued = 0;
8386247Sraf 		qrp->qr_rtcount--;
8396247Sraf 	}
8406247Sraf 	next = ulwp->ul_link;
8416247Sraf 	*ulwpp = next;
8426247Sraf 	ulwp->ul_link = NULL;
8436247Sraf 	if (qrp->qr_tail == ulwp)
8446247Sraf 		qrp->qr_tail = prev;
8456247Sraf 	if (qrp == &ulwp->ul_queue_root) {
8466247Sraf 		/*
8476247Sraf 		 * We can't continue to use the unlinked thread's
8486247Sraf 		 * queue root for the linkage.
8496247Sraf 		 */
8506247Sraf 		queue_root_t *qr_next = qrp->qr_next;
8516247Sraf 		queue_root_t *qr_prev = qrp->qr_prev;
8526247Sraf 
8536247Sraf 		if (qrp->qr_tail) {
8546247Sraf 			/* switch to using the last thread's queue root */
8556247Sraf 			ASSERT(qrp->qr_qlen != 0);
8566247Sraf 			nqrp = &qrp->qr_tail->ul_queue_root;
8576247Sraf 			*nqrp = *qrp;
8586247Sraf 			if (qr_next)
8596247Sraf 				qr_next->qr_prev = nqrp;
8606247Sraf 			if (qr_prev)
8616247Sraf 				qr_prev->qr_next = nqrp;
8626247Sraf 			else
8636247Sraf 				qp->qh_hlist = nqrp;
8646247Sraf 			qp->qh_root = nqrp;
8656247Sraf 		} else {
8666247Sraf 			/* empty queue root; just delete from the hash list */
8676247Sraf 			ASSERT(qrp->qr_qlen == 0);
8686247Sraf 			if (qr_next)
8696247Sraf 				qr_next->qr_prev = qr_prev;
8706247Sraf 			if (qr_prev)
8716247Sraf 				qr_prev->qr_next = qr_next;
8726247Sraf 			else
8736247Sraf 				qp->qh_hlist = qr_next;
8746247Sraf 			qp->qh_root = NULL;
8756247Sraf 			DECR(qp->qh_hlen);
8766247Sraf 		}
8776247Sraf 	}
8786247Sraf }
8796247Sraf 
8806247Sraf ulwp_t *
dequeue(queue_head_t * qp,int * more)8816247Sraf dequeue(queue_head_t *qp, int *more)
8820Sstevel@tonic-gate {
8830Sstevel@tonic-gate 	ulwp_t **ulwpp;
8840Sstevel@tonic-gate 	ulwp_t *ulwp;
8856247Sraf 	ulwp_t *prev;
8866247Sraf 
8876247Sraf 	if ((ulwpp = queue_slot(qp, &prev, more)) == NULL)
8880Sstevel@tonic-gate 		return (NULL);
8890Sstevel@tonic-gate 	ulwp = *ulwpp;
8906247Sraf 	queue_unlink(qp, ulwpp, prev);
8910Sstevel@tonic-gate 	ulwp->ul_sleepq = NULL;
8920Sstevel@tonic-gate 	ulwp->ul_wchan = NULL;
8930Sstevel@tonic-gate 	return (ulwp);
8940Sstevel@tonic-gate }
8950Sstevel@tonic-gate 
8960Sstevel@tonic-gate /*
8970Sstevel@tonic-gate  * Return a pointer to the highest priority thread sleeping on wchan.
8980Sstevel@tonic-gate  */
8990Sstevel@tonic-gate ulwp_t *
queue_waiter(queue_head_t * qp)9006247Sraf queue_waiter(queue_head_t *qp)
9010Sstevel@tonic-gate {
9020Sstevel@tonic-gate 	ulwp_t **ulwpp;
9036247Sraf 	ulwp_t *prev;
9046247Sraf 	int more;
9056247Sraf 
9066247Sraf 	if ((ulwpp = queue_slot(qp, &prev, &more)) == NULL)
9070Sstevel@tonic-gate 		return (NULL);
9080Sstevel@tonic-gate 	return (*ulwpp);
9090Sstevel@tonic-gate }
9100Sstevel@tonic-gate 
9116247Sraf int
dequeue_self(queue_head_t * qp)9126247Sraf dequeue_self(queue_head_t *qp)
9130Sstevel@tonic-gate {
9140Sstevel@tonic-gate 	ulwp_t *self = curthread;
9156247Sraf 	queue_root_t *qrp;
9160Sstevel@tonic-gate 	ulwp_t **ulwpp;
9170Sstevel@tonic-gate 	ulwp_t *ulwp;
9186247Sraf 	ulwp_t *prev;
9190Sstevel@tonic-gate 	int found = 0;
9200Sstevel@tonic-gate 
9210Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
9220Sstevel@tonic-gate 
9230Sstevel@tonic-gate 	/* find self on the sleep queue */
9246247Sraf 	if ((qrp = qp->qh_root) != NULL) {
9256247Sraf 		for (prev = NULL, ulwpp = &qrp->qr_head;
9266247Sraf 		    (ulwp = *ulwpp) != NULL;
9276247Sraf 		    prev = ulwp, ulwpp = &ulwp->ul_link) {
9286247Sraf 			if (ulwp == self) {
9296247Sraf 				queue_unlink(qp, ulwpp, prev);
9306247Sraf 				self->ul_cvmutex = NULL;
9316247Sraf 				self->ul_sleepq = NULL;
9326247Sraf 				self->ul_wchan = NULL;
9336247Sraf 				found = 1;
9346247Sraf 				break;
9356247Sraf 			}
9360Sstevel@tonic-gate 		}
9370Sstevel@tonic-gate 	}
9380Sstevel@tonic-gate 
9390Sstevel@tonic-gate 	if (!found)
9400Sstevel@tonic-gate 		thr_panic("dequeue_self(): curthread not found on queue");
9410Sstevel@tonic-gate 
9426247Sraf 	return ((qrp = qp->qh_root) != NULL && qrp->qr_head != NULL);
9430Sstevel@tonic-gate }
9440Sstevel@tonic-gate 
9450Sstevel@tonic-gate /*
9460Sstevel@tonic-gate  * Called from call_user_handler() and _thrp_suspend() to take
9470Sstevel@tonic-gate  * ourself off of our sleep queue so we can grab locks.
9480Sstevel@tonic-gate  */
9490Sstevel@tonic-gate void
unsleep_self(void)9500Sstevel@tonic-gate unsleep_self(void)
9510Sstevel@tonic-gate {
9520Sstevel@tonic-gate 	ulwp_t *self = curthread;
9530Sstevel@tonic-gate 	queue_head_t *qp;
9540Sstevel@tonic-gate 
9550Sstevel@tonic-gate 	/*
9560Sstevel@tonic-gate 	 * Calling enter_critical()/exit_critical() here would lead
9570Sstevel@tonic-gate 	 * to recursion.  Just manipulate self->ul_critical directly.
9580Sstevel@tonic-gate 	 */
9590Sstevel@tonic-gate 	self->ul_critical++;
9600Sstevel@tonic-gate 	while (self->ul_sleepq != NULL) {
9610Sstevel@tonic-gate 		qp = queue_lock(self->ul_wchan, self->ul_qtype);
9620Sstevel@tonic-gate 		/*
9630Sstevel@tonic-gate 		 * We may have been moved from a CV queue to a
9640Sstevel@tonic-gate 		 * mutex queue while we were attempting queue_lock().
9650Sstevel@tonic-gate 		 * If so, just loop around and try again.
9660Sstevel@tonic-gate 		 * dequeue_self() clears self->ul_sleepq.
9670Sstevel@tonic-gate 		 */
9686247Sraf 		if (qp == self->ul_sleepq)
9696247Sraf 			(void) dequeue_self(qp);
9700Sstevel@tonic-gate 		queue_unlock(qp);
9710Sstevel@tonic-gate 	}
9726247Sraf 	self->ul_writer = 0;
9730Sstevel@tonic-gate 	self->ul_critical--;
9740Sstevel@tonic-gate }
9750Sstevel@tonic-gate 
9760Sstevel@tonic-gate /*
9770Sstevel@tonic-gate  * Common code for calling the the ___lwp_mutex_timedlock() system call.
9780Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
9790Sstevel@tonic-gate  */
9804574Sraf static int
mutex_lock_kernel(mutex_t * mp,timespec_t * tsp,tdb_mutex_stats_t * msp)9810Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
9820Sstevel@tonic-gate {
9830Sstevel@tonic-gate 	ulwp_t *self = curthread;
9840Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
9854574Sraf 	int mtype = mp->mutex_type;
9860Sstevel@tonic-gate 	hrtime_t begin_sleep;
9874574Sraf 	int acquired;
9880Sstevel@tonic-gate 	int error;
9890Sstevel@tonic-gate 
9900Sstevel@tonic-gate 	self->ul_sp = stkptr();
9910Sstevel@tonic-gate 	self->ul_wchan = mp;
9920Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
9930Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
9940Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
9950Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
9960Sstevel@tonic-gate 	}
9970Sstevel@tonic-gate 	if (msp) {
9980Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
9990Sstevel@tonic-gate 		begin_sleep = gethrtime();
10000Sstevel@tonic-gate 	}
10010Sstevel@tonic-gate 
10020Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
10030Sstevel@tonic-gate 
10040Sstevel@tonic-gate 	for (;;) {
10054574Sraf 		/*
10064574Sraf 		 * A return value of EOWNERDEAD or ELOCKUNMAPPED
10074574Sraf 		 * means we successfully acquired the lock.
10084574Sraf 		 */
1009*10887SRoger.Faulkner@Sun.COM 		if ((error = ___lwp_mutex_timedlock(mp, tsp, self)) != 0 &&
10104574Sraf 		    error != EOWNERDEAD && error != ELOCKUNMAPPED) {
10114574Sraf 			acquired = 0;
10120Sstevel@tonic-gate 			break;
10130Sstevel@tonic-gate 		}
10140Sstevel@tonic-gate 
10154574Sraf 		if (mtype & USYNC_PROCESS) {
10160Sstevel@tonic-gate 			/*
10170Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
10180Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
10190Sstevel@tonic-gate 			 */
10200Sstevel@tonic-gate 			enter_critical(self);
10210Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
10220Sstevel@tonic-gate 				exit_critical(self);
10234574Sraf 				acquired = 1;
10240Sstevel@tonic-gate 				break;
10250Sstevel@tonic-gate 			}
10260Sstevel@tonic-gate 			exit_critical(self);
10270Sstevel@tonic-gate 		} else {
10284574Sraf 			acquired = 1;
10290Sstevel@tonic-gate 			break;
10300Sstevel@tonic-gate 		}
10310Sstevel@tonic-gate 	}
10327907SRoger.Faulkner@Sun.COM 
10330Sstevel@tonic-gate 	if (msp)
10340Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
10350Sstevel@tonic-gate 	self->ul_wchan = NULL;
10360Sstevel@tonic-gate 	self->ul_sp = 0;
10370Sstevel@tonic-gate 
10384574Sraf 	if (acquired) {
1039*10887SRoger.Faulkner@Sun.COM 		ASSERT(mp->mutex_owner == (uintptr_t)self);
10404574Sraf 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
10414574Sraf 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
10424574Sraf 	} else {
10434574Sraf 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
10444574Sraf 		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
10454574Sraf 	}
10464574Sraf 
10470Sstevel@tonic-gate 	return (error);
10480Sstevel@tonic-gate }
10490Sstevel@tonic-gate 
10500Sstevel@tonic-gate /*
10510Sstevel@tonic-gate  * Common code for calling the ___lwp_mutex_trylock() system call.
10520Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
10530Sstevel@tonic-gate  */
10540Sstevel@tonic-gate int
mutex_trylock_kernel(mutex_t * mp)10550Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp)
10560Sstevel@tonic-gate {
10570Sstevel@tonic-gate 	ulwp_t *self = curthread;
10580Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
10594574Sraf 	int mtype = mp->mutex_type;
10600Sstevel@tonic-gate 	int error;
10614574Sraf 	int acquired;
10620Sstevel@tonic-gate 
10630Sstevel@tonic-gate 	for (;;) {
10644574Sraf 		/*
10654574Sraf 		 * A return value of EOWNERDEAD or ELOCKUNMAPPED
10664574Sraf 		 * means we successfully acquired the lock.
10674574Sraf 		 */
1068*10887SRoger.Faulkner@Sun.COM 		if ((error = ___lwp_mutex_trylock(mp, self)) != 0 &&
10694574Sraf 		    error != EOWNERDEAD && error != ELOCKUNMAPPED) {
10704574Sraf 			acquired = 0;
10710Sstevel@tonic-gate 			break;
10720Sstevel@tonic-gate 		}
10730Sstevel@tonic-gate 
10744574Sraf 		if (mtype & USYNC_PROCESS) {
10750Sstevel@tonic-gate 			/*
10760Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
10770Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
10780Sstevel@tonic-gate 			 */
10790Sstevel@tonic-gate 			enter_critical(self);
10800Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
10810Sstevel@tonic-gate 				exit_critical(self);
10824574Sraf 				acquired = 1;
10830Sstevel@tonic-gate 				break;
10840Sstevel@tonic-gate 			}
10850Sstevel@tonic-gate 			exit_critical(self);
10860Sstevel@tonic-gate 		} else {
10874574Sraf 			acquired = 1;
10880Sstevel@tonic-gate 			break;
10890Sstevel@tonic-gate 		}
10900Sstevel@tonic-gate 	}
10910Sstevel@tonic-gate 
10924574Sraf 	if (acquired) {
1093*10887SRoger.Faulkner@Sun.COM 		ASSERT(mp->mutex_owner == (uintptr_t)self);
10944574Sraf 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
10954574Sraf 	} else if (error != EBUSY) {
10964574Sraf 		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
10974574Sraf 	}
10984574Sraf 
10990Sstevel@tonic-gate 	return (error);
11000Sstevel@tonic-gate }
11010Sstevel@tonic-gate 
11020Sstevel@tonic-gate volatile sc_shared_t *
setup_schedctl(void)11030Sstevel@tonic-gate setup_schedctl(void)
11040Sstevel@tonic-gate {
11050Sstevel@tonic-gate 	ulwp_t *self = curthread;
11060Sstevel@tonic-gate 	volatile sc_shared_t *scp;
11070Sstevel@tonic-gate 	sc_shared_t *tmp;
11080Sstevel@tonic-gate 
11090Sstevel@tonic-gate 	if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */
11100Sstevel@tonic-gate 	    !self->ul_vfork &&			/* not a child of vfork() */
11110Sstevel@tonic-gate 	    !self->ul_schedctl_called) {	/* haven't been called before */
11120Sstevel@tonic-gate 		enter_critical(self);
11130Sstevel@tonic-gate 		self->ul_schedctl_called = &self->ul_uberdata->uberflags;
11140Sstevel@tonic-gate 		if ((tmp = __schedctl()) != (sc_shared_t *)(-1))
11150Sstevel@tonic-gate 			self->ul_schedctl = scp = tmp;
11160Sstevel@tonic-gate 		exit_critical(self);
11170Sstevel@tonic-gate 	}
11180Sstevel@tonic-gate 	/*
11190Sstevel@tonic-gate 	 * Unless the call to setup_schedctl() is surrounded
11200Sstevel@tonic-gate 	 * by enter_critical()/exit_critical(), the address
11210Sstevel@tonic-gate 	 * we are returning could be invalid due to a forkall()
11220Sstevel@tonic-gate 	 * having occurred in another thread.
11230Sstevel@tonic-gate 	 */
11240Sstevel@tonic-gate 	return (scp);
11250Sstevel@tonic-gate }
11260Sstevel@tonic-gate 
11270Sstevel@tonic-gate /*
11280Sstevel@tonic-gate  * Interfaces from libsched, incorporated into libc.
11290Sstevel@tonic-gate  * libsched.so.1 is now a filter library onto libc.
11300Sstevel@tonic-gate  */
11316812Sraf #pragma weak schedctl_lookup = schedctl_init
11320Sstevel@tonic-gate schedctl_t *
schedctl_init(void)11336812Sraf schedctl_init(void)
11340Sstevel@tonic-gate {
11350Sstevel@tonic-gate 	volatile sc_shared_t *scp = setup_schedctl();
11360Sstevel@tonic-gate 	return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl);
11370Sstevel@tonic-gate }
11380Sstevel@tonic-gate 
11390Sstevel@tonic-gate void
schedctl_exit(void)11406812Sraf schedctl_exit(void)
11410Sstevel@tonic-gate {
11420Sstevel@tonic-gate }
11430Sstevel@tonic-gate 
11440Sstevel@tonic-gate /*
11450Sstevel@tonic-gate  * Contract private interface for java.
11460Sstevel@tonic-gate  * Set up the schedctl data if it doesn't exist yet.
11470Sstevel@tonic-gate  * Return a pointer to the pointer to the schedctl data.
11480Sstevel@tonic-gate  */
11490Sstevel@tonic-gate volatile sc_shared_t *volatile *
_thr_schedctl(void)11500Sstevel@tonic-gate _thr_schedctl(void)
11510Sstevel@tonic-gate {
11520Sstevel@tonic-gate 	ulwp_t *self = curthread;
11530Sstevel@tonic-gate 	volatile sc_shared_t *volatile *ptr;
11540Sstevel@tonic-gate 
11550Sstevel@tonic-gate 	if (self->ul_vfork)
11560Sstevel@tonic-gate 		return (NULL);
11570Sstevel@tonic-gate 	if (*(ptr = &self->ul_schedctl) == NULL)
11580Sstevel@tonic-gate 		(void) setup_schedctl();
11590Sstevel@tonic-gate 	return (ptr);
11600Sstevel@tonic-gate }
11610Sstevel@tonic-gate 
11620Sstevel@tonic-gate /*
11630Sstevel@tonic-gate  * Block signals and attempt to block preemption.
11640Sstevel@tonic-gate  * no_preempt()/preempt() must be used in pairs but can be nested.
11650Sstevel@tonic-gate  */
11660Sstevel@tonic-gate void
no_preempt(ulwp_t * self)11670Sstevel@tonic-gate no_preempt(ulwp_t *self)
11680Sstevel@tonic-gate {
11690Sstevel@tonic-gate 	volatile sc_shared_t *scp;
11700Sstevel@tonic-gate 
11710Sstevel@tonic-gate 	if (self->ul_preempt++ == 0) {
11720Sstevel@tonic-gate 		enter_critical(self);
11730Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL ||
11740Sstevel@tonic-gate 		    (scp = setup_schedctl()) != NULL) {
11750Sstevel@tonic-gate 			/*
11760Sstevel@tonic-gate 			 * Save the pre-existing preempt value.
11770Sstevel@tonic-gate 			 */
11780Sstevel@tonic-gate 			self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt;
11790Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = 1;
11800Sstevel@tonic-gate 		}
11810Sstevel@tonic-gate 	}
11820Sstevel@tonic-gate }
11830Sstevel@tonic-gate 
11840Sstevel@tonic-gate /*
11850Sstevel@tonic-gate  * Undo the effects of no_preempt().
11860Sstevel@tonic-gate  */
11870Sstevel@tonic-gate void
preempt(ulwp_t * self)11880Sstevel@tonic-gate preempt(ulwp_t *self)
11890Sstevel@tonic-gate {
11900Sstevel@tonic-gate 	volatile sc_shared_t *scp;
11910Sstevel@tonic-gate 
11920Sstevel@tonic-gate 	ASSERT(self->ul_preempt > 0);
11930Sstevel@tonic-gate 	if (--self->ul_preempt == 0) {
11940Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL) {
11950Sstevel@tonic-gate 			/*
11960Sstevel@tonic-gate 			 * Restore the pre-existing preempt value.
11970Sstevel@tonic-gate 			 */
11980Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt;
11990Sstevel@tonic-gate 			if (scp->sc_preemptctl.sc_yield &&
12000Sstevel@tonic-gate 			    scp->sc_preemptctl.sc_nopreempt == 0) {
12016515Sraf 				yield();
12020Sstevel@tonic-gate 				if (scp->sc_preemptctl.sc_yield) {
12030Sstevel@tonic-gate 					/*
12040Sstevel@tonic-gate 					 * Shouldn't happen.  This is either
12050Sstevel@tonic-gate 					 * a race condition or the thread
12060Sstevel@tonic-gate 					 * just entered the real-time class.
12070Sstevel@tonic-gate 					 */
12086515Sraf 					yield();
12090Sstevel@tonic-gate 					scp->sc_preemptctl.sc_yield = 0;
12100Sstevel@tonic-gate 				}
12110Sstevel@tonic-gate 			}
12120Sstevel@tonic-gate 		}
12130Sstevel@tonic-gate 		exit_critical(self);
12140Sstevel@tonic-gate 	}
12150Sstevel@tonic-gate }
12160Sstevel@tonic-gate 
12170Sstevel@tonic-gate /*
12180Sstevel@tonic-gate  * If a call to preempt() would cause the current thread to yield or to
12190Sstevel@tonic-gate  * take deferred actions in exit_critical(), then unpark the specified
12200Sstevel@tonic-gate  * lwp so it can run while we delay.  Return the original lwpid if the
12210Sstevel@tonic-gate  * unpark was not performed, else return zero.  The tests are a repeat
12220Sstevel@tonic-gate  * of some of the tests in preempt(), above.  This is a statistical
12230Sstevel@tonic-gate  * optimization solely for cond_sleep_queue(), below.
12240Sstevel@tonic-gate  */
12250Sstevel@tonic-gate static lwpid_t
preempt_unpark(ulwp_t * self,lwpid_t lwpid)12260Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid)
12270Sstevel@tonic-gate {
12280Sstevel@tonic-gate 	volatile sc_shared_t *scp = self->ul_schedctl;
12290Sstevel@tonic-gate 
12300Sstevel@tonic-gate 	ASSERT(self->ul_preempt == 1 && self->ul_critical > 0);
12310Sstevel@tonic-gate 	if ((scp != NULL && scp->sc_preemptctl.sc_yield) ||
12320Sstevel@tonic-gate 	    (self->ul_curplease && self->ul_critical == 1)) {
12330Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
12340Sstevel@tonic-gate 		lwpid = 0;
12350Sstevel@tonic-gate 	}
12360Sstevel@tonic-gate 	return (lwpid);
12370Sstevel@tonic-gate }
12380Sstevel@tonic-gate 
12390Sstevel@tonic-gate /*
12404613Sraf  * Spin for a while (if 'tryhard' is true), trying to grab the lock.
12410Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
12420Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread.
12430Sstevel@tonic-gate  */
12444574Sraf static int
mutex_trylock_adaptive(mutex_t * mp,int tryhard)12454613Sraf mutex_trylock_adaptive(mutex_t *mp, int tryhard)
12460Sstevel@tonic-gate {
12470Sstevel@tonic-gate 	ulwp_t *self = curthread;
12484574Sraf 	int error = EBUSY;
12490Sstevel@tonic-gate 	ulwp_t *ulwp;
12500Sstevel@tonic-gate 	volatile sc_shared_t *scp;
12515629Sraf 	volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
12525629Sraf 	volatile uint64_t *ownerp = (volatile uint64_t *)&mp->mutex_owner;
12535629Sraf 	uint32_t new_lockword;
12545629Sraf 	int count = 0;
12555629Sraf 	int max_count;
12565629Sraf 	uint8_t max_spinners;
12574574Sraf 
12584574Sraf 	ASSERT(!(mp->mutex_type & USYNC_PROCESS));
12594574Sraf 
12607907SRoger.Faulkner@Sun.COM 	if (MUTEX_OWNED(mp, self))
12610Sstevel@tonic-gate 		return (EBUSY);
12620Sstevel@tonic-gate 
12637907SRoger.Faulkner@Sun.COM 	enter_critical(self);
12647907SRoger.Faulkner@Sun.COM 
12654574Sraf 	/* short-cut, not definitive (see below) */
12664574Sraf 	if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
12674574Sraf 		ASSERT(mp->mutex_type & LOCK_ROBUST);
12685629Sraf 		error = ENOTRECOVERABLE;
12695629Sraf 		goto done;
12704574Sraf 	}
12714574Sraf 
12725629Sraf 	/*
12735629Sraf 	 * Make one attempt to acquire the lock before
12745629Sraf 	 * incurring the overhead of the spin loop.
12755629Sraf 	 */
12765629Sraf 	if (set_lock_byte(lockp) == 0) {
12775629Sraf 		*ownerp = (uintptr_t)self;
12785629Sraf 		error = 0;
12795629Sraf 		goto done;
12805629Sraf 	}
12815629Sraf 	if (!tryhard)
12825629Sraf 		goto done;
12835629Sraf 	if (ncpus == 0)
12845629Sraf 		ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
12855629Sraf 	if ((max_spinners = self->ul_max_spinners) >= ncpus)
12865629Sraf 		max_spinners = ncpus - 1;
12875629Sraf 	max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0;
12885629Sraf 	if (max_count == 0)
12895629Sraf 		goto done;
12905629Sraf 
12910Sstevel@tonic-gate 	/*
12920Sstevel@tonic-gate 	 * This spin loop is unfair to lwps that have already dropped into
12930Sstevel@tonic-gate 	 * the kernel to sleep.  They will starve on a highly-contended mutex.
12940Sstevel@tonic-gate 	 * This is just too bad.  The adaptive spin algorithm is intended
12950Sstevel@tonic-gate 	 * to allow programs with highly-contended locks (that is, broken
12960Sstevel@tonic-gate 	 * programs) to execute with reasonable speed despite their contention.
12970Sstevel@tonic-gate 	 * Being fair would reduce the speed of such programs and well-written
12980Sstevel@tonic-gate 	 * programs will not suffer in any case.
12990Sstevel@tonic-gate 	 */
13007907SRoger.Faulkner@Sun.COM 	if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1)
13015629Sraf 		goto done;
13025629Sraf 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
13035629Sraf 	for (count = 1; ; count++) {
13040Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
13050Sstevel@tonic-gate 			*ownerp = (uintptr_t)self;
13064574Sraf 			error = 0;
13074574Sraf 			break;
13080Sstevel@tonic-gate 		}
13095629Sraf 		if (count == max_count)
13105629Sraf 			break;
13110Sstevel@tonic-gate 		SMT_PAUSE();
13120Sstevel@tonic-gate 		/*
13130Sstevel@tonic-gate 		 * Stop spinning if the mutex owner is not running on
13140Sstevel@tonic-gate 		 * a processor; it will not drop the lock any time soon
13150Sstevel@tonic-gate 		 * and we would just be wasting time to keep spinning.
13160Sstevel@tonic-gate 		 *
13170Sstevel@tonic-gate 		 * Note that we are looking at another thread (ulwp_t)
13180Sstevel@tonic-gate 		 * without ensuring that the other thread does not exit.
13190Sstevel@tonic-gate 		 * The scheme relies on ulwp_t structures never being
13200Sstevel@tonic-gate 		 * deallocated by the library (the library employs a free
13210Sstevel@tonic-gate 		 * list of ulwp_t structs that are reused when new threads
13220Sstevel@tonic-gate 		 * are created) and on schedctl shared memory never being
13230Sstevel@tonic-gate 		 * deallocated once created via __schedctl().
13240Sstevel@tonic-gate 		 *
13250Sstevel@tonic-gate 		 * Thus, the worst that can happen when the spinning thread
13260Sstevel@tonic-gate 		 * looks at the owner's schedctl data is that it is looking
13270Sstevel@tonic-gate 		 * at some other thread's schedctl data.  This almost never
13280Sstevel@tonic-gate 		 * happens and is benign when it does.
13290Sstevel@tonic-gate 		 */
13300Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
13310Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
13320Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
13330Sstevel@tonic-gate 			break;
13340Sstevel@tonic-gate 	}
13355629Sraf 	new_lockword = spinners_decr(&mp->mutex_lockword);
13365629Sraf 	if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) {
13375629Sraf 		/*
13385629Sraf 		 * We haven't yet acquired the lock, the lock
13395629Sraf 		 * is free, and there are no other spinners.
13405629Sraf 		 * Make one final attempt to acquire the lock.
13415629Sraf 		 *
13425629Sraf 		 * This isn't strictly necessary since mutex_lock_queue()
13435629Sraf 		 * (the next action this thread will take if it doesn't
13445629Sraf 		 * acquire the lock here) makes one attempt to acquire
13455629Sraf 		 * the lock before putting the thread to sleep.
13465629Sraf 		 *
13475629Sraf 		 * If the next action for this thread (on failure here)
13485629Sraf 		 * were not to call mutex_lock_queue(), this would be
13495629Sraf 		 * necessary for correctness, to avoid ending up with an
13505629Sraf 		 * unheld mutex with waiters but no one to wake them up.
13515629Sraf 		 */
13525629Sraf 		if (set_lock_byte(lockp) == 0) {
13535629Sraf 			*ownerp = (uintptr_t)self;
13545629Sraf 			error = 0;
13555629Sraf 		}
13565629Sraf 		count++;
13575629Sraf 	}
13580Sstevel@tonic-gate 
13595629Sraf done:
13604574Sraf 	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
13614574Sraf 		ASSERT(mp->mutex_type & LOCK_ROBUST);
13624574Sraf 		/*
13636057Sraf 		 * We shouldn't own the mutex.
13646057Sraf 		 * Just clear the lock; everyone has already been waked up.
13654574Sraf 		 */
13667907SRoger.Faulkner@Sun.COM 		*ownerp = 0;
13676057Sraf 		(void) clear_lockbyte(&mp->mutex_lockword);
13684574Sraf 		error = ENOTRECOVERABLE;
13694574Sraf 	}
13704574Sraf 
13717907SRoger.Faulkner@Sun.COM 	exit_critical(self);
13727907SRoger.Faulkner@Sun.COM 
13734574Sraf 	if (error) {
13745629Sraf 		if (count) {
13759397SJonathan.Haslam@Sun.COM 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 0, count);
13765629Sraf 		}
13774574Sraf 		if (error != EBUSY) {
13784574Sraf 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
13794574Sraf 		}
13804574Sraf 	} else {
13815629Sraf 		if (count) {
13829397SJonathan.Haslam@Sun.COM 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
13835629Sraf 		}
13844574Sraf 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
13854574Sraf 		if (mp->mutex_flag & LOCK_OWNERDEAD) {
13864574Sraf 			ASSERT(mp->mutex_type & LOCK_ROBUST);
13874574Sraf 			error = EOWNERDEAD;
13884574Sraf 		}
13894574Sraf 	}
13904574Sraf 
13914574Sraf 	return (error);
13920Sstevel@tonic-gate }
13930Sstevel@tonic-gate 
13940Sstevel@tonic-gate /*
13950Sstevel@tonic-gate  * Same as mutex_trylock_adaptive(), except specifically for queue locks.
13960Sstevel@tonic-gate  * The owner field is not set here; the caller (spin_lock_set()) sets it.
13970Sstevel@tonic-gate  */
13984574Sraf static int
mutex_queuelock_adaptive(mutex_t * mp)13990Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp)
14000Sstevel@tonic-gate {
14010Sstevel@tonic-gate 	ulwp_t *ulwp;
14020Sstevel@tonic-gate 	volatile sc_shared_t *scp;
14030Sstevel@tonic-gate 	volatile uint8_t *lockp;
14040Sstevel@tonic-gate 	volatile uint64_t *ownerp;
14050Sstevel@tonic-gate 	int count = curthread->ul_queue_spin;
14060Sstevel@tonic-gate 
14070Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
14080Sstevel@tonic-gate 
14090Sstevel@tonic-gate 	if (count == 0)
14100Sstevel@tonic-gate 		return (EBUSY);
14110Sstevel@tonic-gate 
14120Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
14130Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
14140Sstevel@tonic-gate 	while (--count >= 0) {
14150Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0)
14160Sstevel@tonic-gate 			return (0);
14170Sstevel@tonic-gate 		SMT_PAUSE();
14180Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
14190Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
14200Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
14210Sstevel@tonic-gate 			break;
14220Sstevel@tonic-gate 	}
14230Sstevel@tonic-gate 
14240Sstevel@tonic-gate 	return (EBUSY);
14250Sstevel@tonic-gate }
14260Sstevel@tonic-gate 
14270Sstevel@tonic-gate /*
14280Sstevel@tonic-gate  * Like mutex_trylock_adaptive(), but for process-shared mutexes.
14294613Sraf  * Spin for a while (if 'tryhard' is true), trying to grab the lock.
14300Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
14310Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread
14320Sstevel@tonic-gate  * and mutex_ownerpid set to the current pid.
14330Sstevel@tonic-gate  */
14344574Sraf static int
mutex_trylock_process(mutex_t * mp,int tryhard)14354613Sraf mutex_trylock_process(mutex_t *mp, int tryhard)
14360Sstevel@tonic-gate {
14370Sstevel@tonic-gate 	ulwp_t *self = curthread;
14385629Sraf 	uberdata_t *udp = self->ul_uberdata;
14394574Sraf 	int error = EBUSY;
14406057Sraf 	volatile uint64_t *lockp = (volatile uint64_t *)&mp->mutex_lockword64;
14415629Sraf 	uint32_t new_lockword;
14425629Sraf 	int count = 0;
14435629Sraf 	int max_count;
14445629Sraf 	uint8_t max_spinners;
14454574Sraf 
14467255Sraf #if defined(__sparc) && !defined(_LP64)
14477255Sraf 	/* horrible hack, necessary only on 32-bit sparc */
14487255Sraf 	int fix_alignment_problem =
14497255Sraf 	    (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
14507255Sraf 	    self->ul_misaligned && !(mp->mutex_type & LOCK_ROBUST));
14517255Sraf #endif
14527255Sraf 
14534574Sraf 	ASSERT(mp->mutex_type & USYNC_PROCESS);
14544574Sraf 
14554574Sraf 	if (shared_mutex_held(mp))
14560Sstevel@tonic-gate 		return (EBUSY);
14570Sstevel@tonic-gate 
14587907SRoger.Faulkner@Sun.COM 	enter_critical(self);
14597907SRoger.Faulkner@Sun.COM 
14604574Sraf 	/* short-cut, not definitive (see below) */
14614574Sraf 	if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
14624574Sraf 		ASSERT(mp->mutex_type & LOCK_ROBUST);
14635629Sraf 		error = ENOTRECOVERABLE;
14645629Sraf 		goto done;
14654574Sraf 	}
14664574Sraf 
14675629Sraf 	/*
14685629Sraf 	 * Make one attempt to acquire the lock before
14695629Sraf 	 * incurring the overhead of the spin loop.
14705629Sraf 	 */
14717255Sraf #if defined(__sparc) && !defined(_LP64)
14727255Sraf 	/* horrible hack, necessary only on 32-bit sparc */
14737255Sraf 	if (fix_alignment_problem) {
14747255Sraf 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
14757255Sraf 			mp->mutex_ownerpid = udp->pid;
14767255Sraf 			mp->mutex_owner = (uintptr_t)self;
14777255Sraf 			error = 0;
14787255Sraf 			goto done;
14797255Sraf 		}
14807255Sraf 	} else
14817255Sraf #endif
14826057Sraf 	if (set_lock_byte64(lockp, udp->pid) == 0) {
14835629Sraf 		mp->mutex_owner = (uintptr_t)self;
14846057Sraf 		/* mp->mutex_ownerpid was set by set_lock_byte64() */
14855629Sraf 		error = 0;
14865629Sraf 		goto done;
14875629Sraf 	}
14885629Sraf 	if (!tryhard)
14895629Sraf 		goto done;
14904574Sraf 	if (ncpus == 0)
14914574Sraf 		ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
14925629Sraf 	if ((max_spinners = self->ul_max_spinners) >= ncpus)
14935629Sraf 		max_spinners = ncpus - 1;
14945629Sraf 	max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0;
14955629Sraf 	if (max_count == 0)
14965629Sraf 		goto done;
14975629Sraf 
14980Sstevel@tonic-gate 	/*
14990Sstevel@tonic-gate 	 * This is a process-shared mutex.
15000Sstevel@tonic-gate 	 * We cannot know if the owner is running on a processor.
15010Sstevel@tonic-gate 	 * We just spin and hope that it is on a processor.
15020Sstevel@tonic-gate 	 */
15037907SRoger.Faulkner@Sun.COM 	if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1)
15045629Sraf 		goto done;
15055629Sraf 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
15065629Sraf 	for (count = 1; ; count++) {
15077255Sraf #if defined(__sparc) && !defined(_LP64)
15087255Sraf 		/* horrible hack, necessary only on 32-bit sparc */
15097255Sraf 		if (fix_alignment_problem) {
15107255Sraf 			if ((*lockp & LOCKMASK64) == 0 &&
15117255Sraf 			    set_lock_byte(&mp->mutex_lockw) == 0) {
15127255Sraf 				mp->mutex_ownerpid = udp->pid;
15137255Sraf 				mp->mutex_owner = (uintptr_t)self;
15147255Sraf 				error = 0;
15157255Sraf 				break;
15167255Sraf 			}
15177255Sraf 		} else
15187255Sraf #endif
15196057Sraf 		if ((*lockp & LOCKMASK64) == 0 &&
15206057Sraf 		    set_lock_byte64(lockp, udp->pid) == 0) {
15214574Sraf 			mp->mutex_owner = (uintptr_t)self;
15226057Sraf 			/* mp->mutex_ownerpid was set by set_lock_byte64() */
15234574Sraf 			error = 0;
15244574Sraf 			break;
15254574Sraf 		}
15265629Sraf 		if (count == max_count)
15275629Sraf 			break;
15284574Sraf 		SMT_PAUSE();
15294574Sraf 	}
15305629Sraf 	new_lockword = spinners_decr(&mp->mutex_lockword);
15315629Sraf 	if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) {
15325629Sraf 		/*
15335629Sraf 		 * We haven't yet acquired the lock, the lock
15345629Sraf 		 * is free, and there are no other spinners.
15355629Sraf 		 * Make one final attempt to acquire the lock.
15365629Sraf 		 *
15375629Sraf 		 * This isn't strictly necessary since mutex_lock_kernel()
15385629Sraf 		 * (the next action this thread will take if it doesn't
15395629Sraf 		 * acquire the lock here) makes one attempt to acquire
15405629Sraf 		 * the lock before putting the thread to sleep.
15415629Sraf 		 *
15425629Sraf 		 * If the next action for this thread (on failure here)
15435629Sraf 		 * were not to call mutex_lock_kernel(), this would be
15445629Sraf 		 * necessary for correctness, to avoid ending up with an
15455629Sraf 		 * unheld mutex with waiters but no one to wake them up.
15465629Sraf 		 */
15477255Sraf #if defined(__sparc) && !defined(_LP64)
15487255Sraf 		/* horrible hack, necessary only on 32-bit sparc */
15497255Sraf 		if (fix_alignment_problem) {
15507255Sraf 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
15517255Sraf 				mp->mutex_ownerpid = udp->pid;
15527255Sraf 				mp->mutex_owner = (uintptr_t)self;
15537255Sraf 				error = 0;
15547255Sraf 			}
15557255Sraf 		} else
15567255Sraf #endif
15576057Sraf 		if (set_lock_byte64(lockp, udp->pid) == 0) {
15585629Sraf 			mp->mutex_owner = (uintptr_t)self;
15596057Sraf 			/* mp->mutex_ownerpid was set by set_lock_byte64() */
15605629Sraf 			error = 0;
15615629Sraf 		}
15625629Sraf 		count++;
15635629Sraf 	}
15644574Sraf 
15655629Sraf done:
15664574Sraf 	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
15674574Sraf 		ASSERT(mp->mutex_type & LOCK_ROBUST);
15684574Sraf 		/*
15696057Sraf 		 * We shouldn't own the mutex.
15706057Sraf 		 * Just clear the lock; everyone has already been waked up.
15714574Sraf 		 */
15724574Sraf 		mp->mutex_owner = 0;
15736057Sraf 		/* mp->mutex_ownerpid is cleared by clear_lockbyte64() */
15746057Sraf 		(void) clear_lockbyte64(&mp->mutex_lockword64);
15754574Sraf 		error = ENOTRECOVERABLE;
15760Sstevel@tonic-gate 	}
15770Sstevel@tonic-gate 
15787907SRoger.Faulkner@Sun.COM 	exit_critical(self);
15797907SRoger.Faulkner@Sun.COM 
15804574Sraf 	if (error) {
15815629Sraf 		if (count) {
15829397SJonathan.Haslam@Sun.COM 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 0, count);
15835629Sraf 		}
15844574Sraf 		if (error != EBUSY) {
15854574Sraf 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
15864574Sraf 		}
15874574Sraf 	} else {
15885629Sraf 		if (count) {
15899397SJonathan.Haslam@Sun.COM 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
15905629Sraf 		}
15914574Sraf 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
15924574Sraf 		if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
15934574Sraf 			ASSERT(mp->mutex_type & LOCK_ROBUST);
15944574Sraf 			if (mp->mutex_flag & LOCK_OWNERDEAD)
15954574Sraf 				error = EOWNERDEAD;
15964574Sraf 			else if (mp->mutex_type & USYNC_PROCESS_ROBUST)
15974574Sraf 				error = ELOCKUNMAPPED;
15984574Sraf 			else
15994574Sraf 				error = EOWNERDEAD;
16004574Sraf 		}
16014574Sraf 	}
16024574Sraf 
16034574Sraf 	return (error);
16040Sstevel@tonic-gate }
16050Sstevel@tonic-gate 
16060Sstevel@tonic-gate /*
16070Sstevel@tonic-gate  * Mutex wakeup code for releasing a USYNC_THREAD mutex.
16080Sstevel@tonic-gate  * Returns the lwpid of the thread that was dequeued, if any.
16090Sstevel@tonic-gate  * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
16100Sstevel@tonic-gate  * to wake up the specified lwp.
16110Sstevel@tonic-gate  */
16124574Sraf static lwpid_t
mutex_wakeup(mutex_t * mp)16130Sstevel@tonic-gate mutex_wakeup(mutex_t *mp)
16140Sstevel@tonic-gate {
16150Sstevel@tonic-gate 	lwpid_t lwpid = 0;
16166247Sraf 	int more;
16170Sstevel@tonic-gate 	queue_head_t *qp;
16180Sstevel@tonic-gate 	ulwp_t *ulwp;
16190Sstevel@tonic-gate 
16200Sstevel@tonic-gate 	/*
16210Sstevel@tonic-gate 	 * Dequeue a waiter from the sleep queue.  Don't touch the mutex
16220Sstevel@tonic-gate 	 * waiters bit if no one was found on the queue because the mutex
16230Sstevel@tonic-gate 	 * might have been deallocated or reallocated for another purpose.
16240Sstevel@tonic-gate 	 */
16250Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
16266247Sraf 	if ((ulwp = dequeue(qp, &more)) != NULL) {
16270Sstevel@tonic-gate 		lwpid = ulwp->ul_lwpid;
16286247Sraf 		mp->mutex_waiters = more;
16290Sstevel@tonic-gate 	}
16300Sstevel@tonic-gate 	queue_unlock(qp);
16310Sstevel@tonic-gate 	return (lwpid);
16320Sstevel@tonic-gate }
16330Sstevel@tonic-gate 
16340Sstevel@tonic-gate /*
16354574Sraf  * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex.
16364574Sraf  */
16374574Sraf static void
mutex_wakeup_all(mutex_t * mp)16384574Sraf mutex_wakeup_all(mutex_t *mp)
16394574Sraf {
16404574Sraf 	queue_head_t *qp;
16416247Sraf 	queue_root_t *qrp;
16424574Sraf 	int nlwpid = 0;
16434574Sraf 	int maxlwps = MAXLWPS;
16444574Sraf 	ulwp_t *ulwp;
16454574Sraf 	lwpid_t buffer[MAXLWPS];
16464574Sraf 	lwpid_t *lwpid = buffer;
16474574Sraf 
16484574Sraf 	/*
16494574Sraf 	 * Walk the list of waiters and prepare to wake up all of them.
16504574Sraf 	 * The waiters flag has already been cleared from the mutex.
16514574Sraf 	 *
16524574Sraf 	 * We keep track of lwpids that are to be unparked in lwpid[].
16534574Sraf 	 * __lwp_unpark_all() is called to unpark all of them after
16544574Sraf 	 * they have been removed from the sleep queue and the sleep
16554574Sraf 	 * queue lock has been dropped.  If we run out of space in our
16564574Sraf 	 * on-stack buffer, we need to allocate more but we can't call
16574574Sraf 	 * lmalloc() because we are holding a queue lock when the overflow
16584574Sraf 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
16594574Sraf 	 * either because the application may have allocated a small
16604574Sraf 	 * stack and we don't want to overrun the stack.  So we call
16614574Sraf 	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
16624574Sraf 	 * system call directly since that path acquires no locks.
16634574Sraf 	 */
16644574Sraf 	qp = queue_lock(mp, MX);
16656247Sraf 	for (;;) {
16666247Sraf 		if ((qrp = qp->qh_root) == NULL ||
16676247Sraf 		    (ulwp = qrp->qr_head) == NULL)
16686247Sraf 			break;
16696247Sraf 		ASSERT(ulwp->ul_wchan == mp);
16706247Sraf 		queue_unlink(qp, &qrp->qr_head, NULL);
16716247Sraf 		ulwp->ul_sleepq = NULL;
16726247Sraf 		ulwp->ul_wchan = NULL;
16736247Sraf 		if (nlwpid == maxlwps)
16746247Sraf 			lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
16756247Sraf 		lwpid[nlwpid++] = ulwp->ul_lwpid;
16764574Sraf 	}
16774574Sraf 
16784574Sraf 	if (nlwpid == 0) {
16794574Sraf 		queue_unlock(qp);
16804574Sraf 	} else {
16815629Sraf 		mp->mutex_waiters = 0;
16824574Sraf 		no_preempt(curthread);
16834574Sraf 		queue_unlock(qp);
16844574Sraf 		if (nlwpid == 1)
16854574Sraf 			(void) __lwp_unpark(lwpid[0]);
16864574Sraf 		else
16874574Sraf 			(void) __lwp_unpark_all(lwpid, nlwpid);
16884574Sraf 		preempt(curthread);
16894574Sraf 	}
16904574Sraf 
16914574Sraf 	if (lwpid != buffer)
16926515Sraf 		(void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
16934574Sraf }
16944574Sraf 
16954574Sraf /*
16965629Sraf  * Release a process-private mutex.
16975629Sraf  * As an optimization, if there are waiters but there are also spinners
16985629Sraf  * attempting to acquire the mutex, then don't bother waking up a waiter;
16995629Sraf  * one of the spinners will acquire the mutex soon and it would be a waste
17005629Sraf  * of resources to wake up some thread just to have it spin for a while
17015629Sraf  * and then possibly go back to sleep.  See mutex_trylock_adaptive().
17020Sstevel@tonic-gate  */
17034574Sraf static lwpid_t
mutex_unlock_queue(mutex_t * mp,int release_all)17044574Sraf mutex_unlock_queue(mutex_t *mp, int release_all)
17050Sstevel@tonic-gate {
17067907SRoger.Faulkner@Sun.COM 	ulwp_t *self = curthread;
17075629Sraf 	lwpid_t lwpid = 0;
17085629Sraf 	uint32_t old_lockword;
17095629Sraf 
17106057Sraf 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
17117907SRoger.Faulkner@Sun.COM 	sigoff(self);
17125629Sraf 	mp->mutex_owner = 0;
17135629Sraf 	old_lockword = clear_lockbyte(&mp->mutex_lockword);
17145629Sraf 	if ((old_lockword & WAITERMASK) &&
17155629Sraf 	    (release_all || (old_lockword & SPINNERMASK) == 0)) {
17160Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
17175629Sraf 		if (release_all)
17185629Sraf 			mutex_wakeup_all(mp);
17195629Sraf 		else
17205629Sraf 			lwpid = mutex_wakeup(mp);
17215629Sraf 		if (lwpid == 0)
17225629Sraf 			preempt(self);
17234574Sraf 	}
17247907SRoger.Faulkner@Sun.COM 	sigon(self);
17250Sstevel@tonic-gate 	return (lwpid);
17260Sstevel@tonic-gate }
17270Sstevel@tonic-gate 
17280Sstevel@tonic-gate /*
17290Sstevel@tonic-gate  * Like mutex_unlock_queue(), but for process-shared mutexes.
17300Sstevel@tonic-gate  */
17314574Sraf static void
mutex_unlock_process(mutex_t * mp,int release_all)17324574Sraf mutex_unlock_process(mutex_t *mp, int release_all)
17330Sstevel@tonic-gate {
17347255Sraf 	ulwp_t *self = curthread;
17356057Sraf 	uint64_t old_lockword64;
17366057Sraf 
17376057Sraf 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
17387907SRoger.Faulkner@Sun.COM 	sigoff(self);
17390Sstevel@tonic-gate 	mp->mutex_owner = 0;
17407255Sraf #if defined(__sparc) && !defined(_LP64)
17417255Sraf 	/* horrible hack, necessary only on 32-bit sparc */
17427255Sraf 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
17437255Sraf 	    self->ul_misaligned && !(mp->mutex_type & LOCK_ROBUST)) {
17447255Sraf 		uint32_t old_lockword;
17457255Sraf 		mp->mutex_ownerpid = 0;
17467255Sraf 		old_lockword = clear_lockbyte(&mp->mutex_lockword);
17477255Sraf 		if ((old_lockword & WAITERMASK) &&
17487255Sraf 		    (release_all || (old_lockword & SPINNERMASK) == 0)) {
17497255Sraf 			no_preempt(self);	/* ensure a prompt wakeup */
17507255Sraf 			(void) ___lwp_mutex_wakeup(mp, release_all);
17517255Sraf 			preempt(self);
17527255Sraf 		}
17537907SRoger.Faulkner@Sun.COM 		sigon(self);
17547255Sraf 		return;
17557255Sraf 	}
17567255Sraf #endif
17576057Sraf 	/* mp->mutex_ownerpid is cleared by clear_lockbyte64() */
17586057Sraf 	old_lockword64 = clear_lockbyte64(&mp->mutex_lockword64);
17596057Sraf 	if ((old_lockword64 & WAITERMASK64) &&
17606057Sraf 	    (release_all || (old_lockword64 & SPINNERMASK64) == 0)) {
17615629Sraf 		no_preempt(self);	/* ensure a prompt wakeup */
17625629Sraf 		(void) ___lwp_mutex_wakeup(mp, release_all);
17635629Sraf 		preempt(self);
17640Sstevel@tonic-gate 	}
17657907SRoger.Faulkner@Sun.COM 	sigon(self);
17660Sstevel@tonic-gate }
17670Sstevel@tonic-gate 
17680Sstevel@tonic-gate void
stall(void)17690Sstevel@tonic-gate stall(void)
17700Sstevel@tonic-gate {
17710Sstevel@tonic-gate 	for (;;)
17720Sstevel@tonic-gate 		(void) mutex_lock_kernel(&stall_mutex, NULL, NULL);
17730Sstevel@tonic-gate }
17740Sstevel@tonic-gate 
17750Sstevel@tonic-gate /*
17760Sstevel@tonic-gate  * Acquire a USYNC_THREAD mutex via user-level sleep queues.
17770Sstevel@tonic-gate  * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
17784574Sraf  * If successful, returns with mutex_owner set correctly.
17790Sstevel@tonic-gate  */
17800Sstevel@tonic-gate int
mutex_lock_queue(ulwp_t * self,tdb_mutex_stats_t * msp,mutex_t * mp,timespec_t * tsp)17810Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
17820Sstevel@tonic-gate 	timespec_t *tsp)
17830Sstevel@tonic-gate {
17840Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
17850Sstevel@tonic-gate 	queue_head_t *qp;
17860Sstevel@tonic-gate 	hrtime_t begin_sleep;
17870Sstevel@tonic-gate 	int error = 0;
17880Sstevel@tonic-gate 
17890Sstevel@tonic-gate 	self->ul_sp = stkptr();
17900Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
17910Sstevel@tonic-gate 		self->ul_wchan = mp;
17920Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
17930Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
17940Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
17950Sstevel@tonic-gate 	}
17960Sstevel@tonic-gate 	if (msp) {
17970Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
17980Sstevel@tonic-gate 		begin_sleep = gethrtime();
17990Sstevel@tonic-gate 	}
18000Sstevel@tonic-gate 
18010Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
18020Sstevel@tonic-gate 
18030Sstevel@tonic-gate 	/*
18040Sstevel@tonic-gate 	 * Put ourself on the sleep queue, and while we are
18050Sstevel@tonic-gate 	 * unable to grab the lock, go park in the kernel.
18060Sstevel@tonic-gate 	 * Take ourself off the sleep queue after we acquire the lock.
18070Sstevel@tonic-gate 	 * The waiter bit can be set/cleared only while holding the queue lock.
18080Sstevel@tonic-gate 	 */
18090Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
18106247Sraf 	enqueue(qp, self, 0);
18110Sstevel@tonic-gate 	mp->mutex_waiters = 1;
18120Sstevel@tonic-gate 	for (;;) {
18130Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
18140Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
18156247Sraf 			mp->mutex_waiters = dequeue_self(qp);
18160Sstevel@tonic-gate 			break;
18170Sstevel@tonic-gate 		}
18180Sstevel@tonic-gate 		set_parking_flag(self, 1);
18190Sstevel@tonic-gate 		queue_unlock(qp);
18200Sstevel@tonic-gate 		/*
18210Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
18220Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
18230Sstevel@tonic-gate 		 */
18245629Sraf 		error = __lwp_park(tsp, 0);
18250Sstevel@tonic-gate 		set_parking_flag(self, 0);
18260Sstevel@tonic-gate 		/*
18270Sstevel@tonic-gate 		 * We could have taken a signal or suspended ourself.
18280Sstevel@tonic-gate 		 * If we did, then we removed ourself from the queue.
18290Sstevel@tonic-gate 		 * Someone else may have removed us from the queue
18300Sstevel@tonic-gate 		 * as a consequence of mutex_unlock().  We may have
18310Sstevel@tonic-gate 		 * gotten a timeout from __lwp_park().  Or we may still
18320Sstevel@tonic-gate 		 * be on the queue and this is just a spurious wakeup.
18330Sstevel@tonic-gate 		 */
18340Sstevel@tonic-gate 		qp = queue_lock(mp, MX);
18350Sstevel@tonic-gate 		if (self->ul_sleepq == NULL) {
18365629Sraf 			if (error) {
18376247Sraf 				mp->mutex_waiters = queue_waiter(qp)? 1 : 0;
18385629Sraf 				if (error != EINTR)
18395629Sraf 					break;
18405629Sraf 				error = 0;
18415629Sraf 			}
18420Sstevel@tonic-gate 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
18430Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
18440Sstevel@tonic-gate 				break;
18450Sstevel@tonic-gate 			}
18466247Sraf 			enqueue(qp, self, 0);
18470Sstevel@tonic-gate 			mp->mutex_waiters = 1;
18480Sstevel@tonic-gate 		}
18490Sstevel@tonic-gate 		ASSERT(self->ul_sleepq == qp &&
18500Sstevel@tonic-gate 		    self->ul_qtype == MX &&
18510Sstevel@tonic-gate 		    self->ul_wchan == mp);
18520Sstevel@tonic-gate 		if (error) {
18535629Sraf 			if (error != EINTR) {
18546247Sraf 				mp->mutex_waiters = dequeue_self(qp);
18555629Sraf 				break;
18565629Sraf 			}
18575629Sraf 			error = 0;
18580Sstevel@tonic-gate 		}
18590Sstevel@tonic-gate 	}
18600Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
18610Sstevel@tonic-gate 	    self->ul_wchan == NULL);
18620Sstevel@tonic-gate 	self->ul_sp = 0;
18630Sstevel@tonic-gate 
18640Sstevel@tonic-gate 	ASSERT(error == 0 || error == EINVAL || error == ETIME);
18654574Sraf 
18664574Sraf 	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
18674574Sraf 		ASSERT(mp->mutex_type & LOCK_ROBUST);
18684574Sraf 		/*
18696057Sraf 		 * We shouldn't own the mutex.
18706057Sraf 		 * Just clear the lock; everyone has already been waked up.
18714574Sraf 		 */
18724574Sraf 		mp->mutex_owner = 0;
18736057Sraf 		(void) clear_lockbyte(&mp->mutex_lockword);
18744574Sraf 		error = ENOTRECOVERABLE;
18754574Sraf 	}
18764574Sraf 
18777907SRoger.Faulkner@Sun.COM 	queue_unlock(qp);
18787907SRoger.Faulkner@Sun.COM 
18797907SRoger.Faulkner@Sun.COM 	if (msp)
18807907SRoger.Faulkner@Sun.COM 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
18817907SRoger.Faulkner@Sun.COM 
18824574Sraf 	if (error) {
18834574Sraf 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
18844574Sraf 		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
18854574Sraf 	} else {
18864574Sraf 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
18874574Sraf 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
18884574Sraf 		if (mp->mutex_flag & LOCK_OWNERDEAD) {
18894574Sraf 			ASSERT(mp->mutex_type & LOCK_ROBUST);
18904574Sraf 			error = EOWNERDEAD;
18914574Sraf 		}
18924574Sraf 	}
18934574Sraf 
18940Sstevel@tonic-gate 	return (error);
18950Sstevel@tonic-gate }
18960Sstevel@tonic-gate 
18974574Sraf static int
mutex_recursion(mutex_t * mp,int mtype,int try)18984574Sraf mutex_recursion(mutex_t *mp, int mtype, int try)
18994574Sraf {
19006812Sraf 	ASSERT(mutex_held(mp));
19014574Sraf 	ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK));
19024574Sraf 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
19034574Sraf 
19044574Sraf 	if (mtype & LOCK_RECURSIVE) {
19054574Sraf 		if (mp->mutex_rcount == RECURSION_MAX) {
19064574Sraf 			DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN);
19074574Sraf 			return (EAGAIN);
19084574Sraf 		}
19094574Sraf 		mp->mutex_rcount++;
19104574Sraf 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
19114574Sraf 		return (0);
19124574Sraf 	}
19134574Sraf 	if (try == MUTEX_LOCK) {
19144574Sraf 		DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
19154574Sraf 		return (EDEADLK);
19164574Sraf 	}
19174574Sraf 	return (EBUSY);
19184574Sraf }
19194574Sraf 
19204574Sraf /*
19214574Sraf  * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so
19224574Sraf  * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary.
19234574Sraf  * We use tdb_hash_lock here and in the synch object tracking code in
19244574Sraf  * the tdb_agent.c file.  There is no conflict between these two usages.
19254574Sraf  */
19264574Sraf void
register_lock(mutex_t * mp)19274574Sraf register_lock(mutex_t *mp)
19284574Sraf {
19294574Sraf 	uberdata_t *udp = curthread->ul_uberdata;
19304574Sraf 	uint_t hash = LOCK_HASH(mp);
19314574Sraf 	robust_t *rlp;
19329170SRoger.Faulkner@Sun.COM 	robust_t *invalid;
19334574Sraf 	robust_t **rlpp;
19344574Sraf 	robust_t **table;
19354574Sraf 
19364574Sraf 	if ((table = udp->robustlocks) == NULL) {
19374574Sraf 		lmutex_lock(&udp->tdb_hash_lock);
19384574Sraf 		if ((table = udp->robustlocks) == NULL) {
19394574Sraf 			table = lmalloc(LOCKHASHSZ * sizeof (robust_t *));
19406812Sraf 			membar_producer();
19414574Sraf 			udp->robustlocks = table;
19424574Sraf 		}
19434574Sraf 		lmutex_unlock(&udp->tdb_hash_lock);
19444574Sraf 	}
19456812Sraf 	membar_consumer();
19464574Sraf 
19474574Sraf 	/*
19484574Sraf 	 * First search the registered table with no locks held.
19494574Sraf 	 * This is safe because the table never shrinks
19504574Sraf 	 * and we can only get a false negative.
19514574Sraf 	 */
19524574Sraf 	for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) {
19534574Sraf 		if (rlp->robust_lock == mp)	/* already registered */
19544574Sraf 			return;
19554574Sraf 	}
19564574Sraf 
19574574Sraf 	/*
19584574Sraf 	 * The lock was not found.
19594574Sraf 	 * Repeat the operation with tdb_hash_lock held.
19604574Sraf 	 */
19614574Sraf 	lmutex_lock(&udp->tdb_hash_lock);
19624574Sraf 
19639170SRoger.Faulkner@Sun.COM 	invalid = NULL;
19644574Sraf 	for (rlpp = &table[hash];
19654574Sraf 	    (rlp = *rlpp) != NULL;
19664574Sraf 	    rlpp = &rlp->robust_next) {
19674574Sraf 		if (rlp->robust_lock == mp) {	/* already registered */
19684574Sraf 			lmutex_unlock(&udp->tdb_hash_lock);
19694574Sraf 			return;
19704574Sraf 		}
19719170SRoger.Faulkner@Sun.COM 		/* remember the first invalid entry, if any */
19729170SRoger.Faulkner@Sun.COM 		if (rlp->robust_lock == INVALID_ADDR && invalid == NULL)
19739170SRoger.Faulkner@Sun.COM 			invalid = rlp;
19744574Sraf 	}
19754574Sraf 
19764574Sraf 	/*
19774574Sraf 	 * The lock has never been registered.
19789170SRoger.Faulkner@Sun.COM 	 * Add it to the table and register it now.
19794574Sraf 	 */
19809264SRoger.Faulkner@Sun.COM 	if ((rlp = invalid) != NULL) {
19819170SRoger.Faulkner@Sun.COM 		/*
19829170SRoger.Faulkner@Sun.COM 		 * Reuse the invalid entry we found above.
19839170SRoger.Faulkner@Sun.COM 		 * The linkages are still correct.
19849170SRoger.Faulkner@Sun.COM 		 */
19859264SRoger.Faulkner@Sun.COM 		rlp->robust_lock = mp;
19869170SRoger.Faulkner@Sun.COM 		membar_producer();
19879170SRoger.Faulkner@Sun.COM 	} else {
19889170SRoger.Faulkner@Sun.COM 		/*
19899170SRoger.Faulkner@Sun.COM 		 * Allocate a new entry and add it to
19909170SRoger.Faulkner@Sun.COM 		 * the hash table and to the global list.
19919170SRoger.Faulkner@Sun.COM 		 */
19929170SRoger.Faulkner@Sun.COM 		rlp = lmalloc(sizeof (*rlp));
19939170SRoger.Faulkner@Sun.COM 		rlp->robust_lock = mp;
19949170SRoger.Faulkner@Sun.COM 		rlp->robust_next = NULL;
19959170SRoger.Faulkner@Sun.COM 		rlp->robust_list = udp->robustlist;
19969170SRoger.Faulkner@Sun.COM 		udp->robustlist = rlp;
19979170SRoger.Faulkner@Sun.COM 		membar_producer();
19989170SRoger.Faulkner@Sun.COM 		*rlpp = rlp;
19999170SRoger.Faulkner@Sun.COM 	}
20009170SRoger.Faulkner@Sun.COM 
20019170SRoger.Faulkner@Sun.COM 	lmutex_unlock(&udp->tdb_hash_lock);
20029170SRoger.Faulkner@Sun.COM 
20039264SRoger.Faulkner@Sun.COM 	(void) ___lwp_mutex_register(mp, &rlp->robust_lock);
20044574Sraf }
20054574Sraf 
20064574Sraf /*
20074574Sraf  * This is called in the child of fork()/forkall() to start over
20084574Sraf  * with a clean slate.  (Each process must register its own locks.)
20094574Sraf  * No locks are needed because all other threads are suspended or gone.
20104574Sraf  */
20114574Sraf void
unregister_locks(void)20129264SRoger.Faulkner@Sun.COM unregister_locks(void)
20134574Sraf {
20144574Sraf 	uberdata_t *udp = curthread->ul_uberdata;
20154574Sraf 	robust_t **table;
20164574Sraf 	robust_t *rlp;
20174574Sraf 	robust_t *next;
20184574Sraf 
20199170SRoger.Faulkner@Sun.COM 	/*
20209170SRoger.Faulkner@Sun.COM 	 * Do this first, before calling lfree().
20219170SRoger.Faulkner@Sun.COM 	 */
20229170SRoger.Faulkner@Sun.COM 	table = udp->robustlocks;
20239170SRoger.Faulkner@Sun.COM 	udp->robustlocks = NULL;
20249170SRoger.Faulkner@Sun.COM 	rlp = udp->robustlist;
20259170SRoger.Faulkner@Sun.COM 	udp->robustlist = NULL;
20269170SRoger.Faulkner@Sun.COM 
20279170SRoger.Faulkner@Sun.COM 	/*
20289264SRoger.Faulkner@Sun.COM 	 * Do this by traversing the global list, not the hash table.
20299170SRoger.Faulkner@Sun.COM 	 */
20309170SRoger.Faulkner@Sun.COM 	while (rlp != NULL) {
20319170SRoger.Faulkner@Sun.COM 		next = rlp->robust_list;
20329170SRoger.Faulkner@Sun.COM 		lfree(rlp, sizeof (*rlp));
20339170SRoger.Faulkner@Sun.COM 		rlp = next;
20349170SRoger.Faulkner@Sun.COM 	}
20359170SRoger.Faulkner@Sun.COM 	if (table != NULL)
20364574Sraf 		lfree(table, LOCKHASHSZ * sizeof (robust_t *));
20374574Sraf }
20384574Sraf 
20390Sstevel@tonic-gate /*
20400Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
20410Sstevel@tonic-gate  */
20426247Sraf int
mutex_lock_internal(mutex_t * mp,timespec_t * tsp,int try)20430Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
20440Sstevel@tonic-gate {
20450Sstevel@tonic-gate 	ulwp_t *self = curthread;
20460Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
20470Sstevel@tonic-gate 	int mtype = mp->mutex_type;
20480Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
20490Sstevel@tonic-gate 	int error = 0;
20506247Sraf 	int noceil = try & MUTEX_NOCEIL;
20514574Sraf 	uint8_t ceil;
20524574Sraf 	int myprio;
20530Sstevel@tonic-gate 
20546247Sraf 	try &= ~MUTEX_NOCEIL;
20550Sstevel@tonic-gate 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
20560Sstevel@tonic-gate 
20570Sstevel@tonic-gate 	if (!self->ul_schedctl_called)
20580Sstevel@tonic-gate 		(void) setup_schedctl();
20590Sstevel@tonic-gate 
20600Sstevel@tonic-gate 	if (msp && try == MUTEX_TRY)
20610Sstevel@tonic-gate 		tdb_incr(msp->mutex_try);
20620Sstevel@tonic-gate 
20636812Sraf 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_held(mp))
20644574Sraf 		return (mutex_recursion(mp, mtype, try));
20650Sstevel@tonic-gate 
20660Sstevel@tonic-gate 	if (self->ul_error_detection && try == MUTEX_LOCK &&
20676812Sraf 	    tsp == NULL && mutex_held(mp))
20680Sstevel@tonic-gate 		lock_error(mp, "mutex_lock", NULL, NULL);
20690Sstevel@tonic-gate 
20706247Sraf 	if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) {
20716247Sraf 		update_sched(self);
20726247Sraf 		if (self->ul_cid != self->ul_rtclassid) {
20736247Sraf 			DTRACE_PROBE2(plockstat, mutex__error, mp, EPERM);
20746247Sraf 			return (EPERM);
20756247Sraf 		}
20764574Sraf 		ceil = mp->mutex_ceiling;
20776247Sraf 		myprio = self->ul_epri? self->ul_epri : self->ul_pri;
20784574Sraf 		if (myprio > ceil) {
20794574Sraf 			DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL);
20804574Sraf 			return (EINVAL);
20814574Sraf 		}
20824574Sraf 		if ((error = _ceil_mylist_add(mp)) != 0) {
20834574Sraf 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
20844574Sraf 			return (error);
20850Sstevel@tonic-gate 		}
20864574Sraf 		if (myprio < ceil)
20874574Sraf 			_ceil_prio_inherit(ceil);
20884574Sraf 	}
20894574Sraf 
20904574Sraf 	if ((mtype & (USYNC_PROCESS | LOCK_ROBUST))
20914574Sraf 	    == (USYNC_PROCESS | LOCK_ROBUST))
20924574Sraf 		register_lock(mp);
20934574Sraf 
20944574Sraf 	if (mtype & LOCK_PRIO_INHERIT) {
20954574Sraf 		/* go straight to the kernel */
20964574Sraf 		if (try == MUTEX_TRY)
20974574Sraf 			error = mutex_trylock_kernel(mp);
20984574Sraf 		else	/* MUTEX_LOCK */
20994574Sraf 			error = mutex_lock_kernel(mp, tsp, msp);
21004574Sraf 		/*
21014574Sraf 		 * The kernel never sets or clears the lock byte
21024574Sraf 		 * for LOCK_PRIO_INHERIT mutexes.
21034574Sraf 		 * Set it here for consistency.
21044574Sraf 		 */
21054574Sraf 		switch (error) {
21064574Sraf 		case 0:
21076247Sraf 			self->ul_pilocks++;
21084574Sraf 			mp->mutex_lockw = LOCKSET;
21094574Sraf 			break;
21104574Sraf 		case EOWNERDEAD:
21114574Sraf 		case ELOCKUNMAPPED:
21126247Sraf 			self->ul_pilocks++;
21134574Sraf 			mp->mutex_lockw = LOCKSET;
21144574Sraf 			/* FALLTHROUGH */
21154574Sraf 		case ENOTRECOVERABLE:
21164574Sraf 			ASSERT(mtype & LOCK_ROBUST);
21174574Sraf 			break;
21184574Sraf 		case EDEADLK:
21197376SRoger.Faulkner@Sun.COM 			if (try == MUTEX_TRY) {
21207376SRoger.Faulkner@Sun.COM 				error = EBUSY;
21217376SRoger.Faulkner@Sun.COM 			} else if (tsp != NULL) {	/* simulate a timeout */
21227376SRoger.Faulkner@Sun.COM 				/*
21237376SRoger.Faulkner@Sun.COM 				 * Note: mutex_timedlock() never returns EINTR.
21247376SRoger.Faulkner@Sun.COM 				 */
21257376SRoger.Faulkner@Sun.COM 				timespec_t ts = *tsp;
21267376SRoger.Faulkner@Sun.COM 				timespec_t rts;
21277376SRoger.Faulkner@Sun.COM 
21287376SRoger.Faulkner@Sun.COM 				while (__nanosleep(&ts, &rts) == EINTR)
21297376SRoger.Faulkner@Sun.COM 					ts = rts;
21307376SRoger.Faulkner@Sun.COM 				error = ETIME;
21317376SRoger.Faulkner@Sun.COM 			} else {		/* simulate a deadlock */
21324574Sraf 				stall();
21337376SRoger.Faulkner@Sun.COM 			}
21344574Sraf 			break;
21350Sstevel@tonic-gate 		}
21360Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
21374613Sraf 		error = mutex_trylock_process(mp, try == MUTEX_LOCK);
21384574Sraf 		if (error == EBUSY && try == MUTEX_LOCK)
21390Sstevel@tonic-gate 			error = mutex_lock_kernel(mp, tsp, msp);
21405629Sraf 	} else {	/* USYNC_THREAD */
21414613Sraf 		error = mutex_trylock_adaptive(mp, try == MUTEX_LOCK);
21424574Sraf 		if (error == EBUSY && try == MUTEX_LOCK)
21434574Sraf 			error = mutex_lock_queue(self, msp, mp, tsp);
21440Sstevel@tonic-gate 	}
21450Sstevel@tonic-gate 
21460Sstevel@tonic-gate 	switch (error) {
21474574Sraf 	case 0:
21480Sstevel@tonic-gate 	case EOWNERDEAD:
21490Sstevel@tonic-gate 	case ELOCKUNMAPPED:
21504574Sraf 		if (mtype & LOCK_ROBUST)
21514574Sraf 			remember_lock(mp);
21520Sstevel@tonic-gate 		if (msp)
21530Sstevel@tonic-gate 			record_begin_hold(msp);
21540Sstevel@tonic-gate 		break;
21550Sstevel@tonic-gate 	default:
21566247Sraf 		if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) {
21574574Sraf 			(void) _ceil_mylist_del(mp);
21584574Sraf 			if (myprio < ceil)
21594574Sraf 				_ceil_prio_waive();
21604574Sraf 		}
21610Sstevel@tonic-gate 		if (try == MUTEX_TRY) {
21620Sstevel@tonic-gate 			if (msp)
21630Sstevel@tonic-gate 				tdb_incr(msp->mutex_try_fail);
21640Sstevel@tonic-gate 			if (__td_event_report(self, TD_LOCK_TRY, udp)) {
21650Sstevel@tonic-gate 				self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
21660Sstevel@tonic-gate 				tdb_event(TD_LOCK_TRY, udp);
21670Sstevel@tonic-gate 			}
21680Sstevel@tonic-gate 		}
21690Sstevel@tonic-gate 		break;
21700Sstevel@tonic-gate 	}
21710Sstevel@tonic-gate 
21720Sstevel@tonic-gate 	return (error);
21730Sstevel@tonic-gate }
21740Sstevel@tonic-gate 
21750Sstevel@tonic-gate int
fast_process_lock(mutex_t * mp,timespec_t * tsp,int mtype,int try)21760Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
21770Sstevel@tonic-gate {
21780Sstevel@tonic-gate 	ulwp_t *self = curthread;
21790Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
21800Sstevel@tonic-gate 
21810Sstevel@tonic-gate 	/*
21820Sstevel@tonic-gate 	 * We know that USYNC_PROCESS is set in mtype and that
21830Sstevel@tonic-gate 	 * zero, one, or both of the flags LOCK_RECURSIVE and
21840Sstevel@tonic-gate 	 * LOCK_ERRORCHECK are set, and that no other flags are set.
21850Sstevel@tonic-gate 	 */
21864574Sraf 	ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0);
21870Sstevel@tonic-gate 	enter_critical(self);
21887255Sraf #if defined(__sparc) && !defined(_LP64)
21897255Sraf 	/* horrible hack, necessary only on 32-bit sparc */
21907255Sraf 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
21917255Sraf 	    self->ul_misaligned) {
21927255Sraf 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
21937255Sraf 			mp->mutex_ownerpid = udp->pid;
21947255Sraf 			mp->mutex_owner = (uintptr_t)self;
21957255Sraf 			exit_critical(self);
21967255Sraf 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
21977255Sraf 			return (0);
21987255Sraf 		}
21997255Sraf 	} else
22007255Sraf #endif
22016057Sraf 	if (set_lock_byte64(&mp->mutex_lockword64, udp->pid) == 0) {
22020Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
22036057Sraf 		/* mp->mutex_ownerpid was set by set_lock_byte64() */
22040Sstevel@tonic-gate 		exit_critical(self);
22050Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
22060Sstevel@tonic-gate 		return (0);
22070Sstevel@tonic-gate 	}
22080Sstevel@tonic-gate 	exit_critical(self);
22090Sstevel@tonic-gate 
22104574Sraf 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp))
22114574Sraf 		return (mutex_recursion(mp, mtype, try));
22124574Sraf 
22134613Sraf 	if (try == MUTEX_LOCK) {
22144613Sraf 		if (mutex_trylock_process(mp, 1) == 0)
22154613Sraf 			return (0);
22160Sstevel@tonic-gate 		return (mutex_lock_kernel(mp, tsp, NULL));
22174613Sraf 	}
22180Sstevel@tonic-gate 
22190Sstevel@tonic-gate 	if (__td_event_report(self, TD_LOCK_TRY, udp)) {
22200Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
22210Sstevel@tonic-gate 		tdb_event(TD_LOCK_TRY, udp);
22220Sstevel@tonic-gate 	}
22230Sstevel@tonic-gate 	return (EBUSY);
22240Sstevel@tonic-gate }
22250Sstevel@tonic-gate 
22260Sstevel@tonic-gate static int
mutex_lock_impl(mutex_t * mp,timespec_t * tsp)22270Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
22280Sstevel@tonic-gate {
22290Sstevel@tonic-gate 	ulwp_t *self = curthread;
22306247Sraf 	int mtype = mp->mutex_type;
22310Sstevel@tonic-gate 	uberflags_t *gflags;
22320Sstevel@tonic-gate 
22337255Sraf 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
22347255Sraf 	    self->ul_error_detection && self->ul_misaligned == 0)
22357255Sraf 		lock_error(mp, "mutex_lock", NULL, "mutex is misaligned");
22367255Sraf 
22370Sstevel@tonic-gate 	/*
22380Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
22390Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
22400Sstevel@tonic-gate 	 * no error detection, no lock statistics,
22410Sstevel@tonic-gate 	 * and the process has only a single thread.
22420Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
22430Sstevel@tonic-gate 	 */
22446247Sraf 	if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
22456247Sraf 	    self->ul_uberdata->uberflags.uf_all) == 0) {
22460Sstevel@tonic-gate 		/*
22470Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
22487907SRoger.Faulkner@Sun.COM 		 * We do, however, need to protect against signals.
22490Sstevel@tonic-gate 		 */
22500Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
22517907SRoger.Faulkner@Sun.COM 			sigoff(self);
22520Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
22530Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
22547907SRoger.Faulkner@Sun.COM 			sigon(self);
22550Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
22560Sstevel@tonic-gate 			return (0);
22570Sstevel@tonic-gate 		}
22584574Sraf 		if (mtype && MUTEX_OWNER(mp) == self)
22594574Sraf 			return (mutex_recursion(mp, mtype, MUTEX_LOCK));
22600Sstevel@tonic-gate 		/*
22610Sstevel@tonic-gate 		 * We have reached a deadlock, probably because the
22620Sstevel@tonic-gate 		 * process is executing non-async-signal-safe code in
22630Sstevel@tonic-gate 		 * a signal handler and is attempting to acquire a lock
22640Sstevel@tonic-gate 		 * that it already owns.  This is not surprising, given
22650Sstevel@tonic-gate 		 * bad programming practices over the years that has
22660Sstevel@tonic-gate 		 * resulted in applications calling printf() and such
22670Sstevel@tonic-gate 		 * in their signal handlers.  Unless the user has told
22680Sstevel@tonic-gate 		 * us that the signal handlers are safe by setting:
22690Sstevel@tonic-gate 		 *	export _THREAD_ASYNC_SAFE=1
22700Sstevel@tonic-gate 		 * we return EDEADLK rather than actually deadlocking.
22710Sstevel@tonic-gate 		 */
22720Sstevel@tonic-gate 		if (tsp == NULL &&
22730Sstevel@tonic-gate 		    MUTEX_OWNER(mp) == self && !self->ul_async_safe) {
22740Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
22750Sstevel@tonic-gate 			return (EDEADLK);
22760Sstevel@tonic-gate 		}
22770Sstevel@tonic-gate 	}
22780Sstevel@tonic-gate 
22790Sstevel@tonic-gate 	/*
22800Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
22810Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
22820Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
22830Sstevel@tonic-gate 	 */
22840Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
22850Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
22860Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
22870Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
22880Sstevel@tonic-gate 			return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
22897907SRoger.Faulkner@Sun.COM 		sigoff(self);
22900Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
22910Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
22927907SRoger.Faulkner@Sun.COM 			sigon(self);
22930Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
22940Sstevel@tonic-gate 			return (0);
22950Sstevel@tonic-gate 		}
22967907SRoger.Faulkner@Sun.COM 		sigon(self);
22974574Sraf 		if (mtype && MUTEX_OWNER(mp) == self)
22984574Sraf 			return (mutex_recursion(mp, mtype, MUTEX_LOCK));
22994613Sraf 		if (mutex_trylock_adaptive(mp, 1) != 0)
23004574Sraf 			return (mutex_lock_queue(self, NULL, mp, tsp));
23014574Sraf 		return (0);
23020Sstevel@tonic-gate 	}
23030Sstevel@tonic-gate 
23040Sstevel@tonic-gate 	/* else do it the long way */
23050Sstevel@tonic-gate 	return (mutex_lock_internal(mp, tsp, MUTEX_LOCK));
23060Sstevel@tonic-gate }
23070Sstevel@tonic-gate 
23086812Sraf #pragma weak pthread_mutex_lock = mutex_lock
23096812Sraf #pragma weak _mutex_lock = mutex_lock
23100Sstevel@tonic-gate int
mutex_lock(mutex_t * mp)23116812Sraf mutex_lock(mutex_t *mp)
23120Sstevel@tonic-gate {
23130Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23140Sstevel@tonic-gate 	return (mutex_lock_impl(mp, NULL));
23150Sstevel@tonic-gate }
23160Sstevel@tonic-gate 
23170Sstevel@tonic-gate int
pthread_mutex_timedlock(pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD abstime)23186812Sraf pthread_mutex_timedlock(pthread_mutex_t *_RESTRICT_KYWD mp,
23196812Sraf 	const struct timespec *_RESTRICT_KYWD abstime)
23200Sstevel@tonic-gate {
23210Sstevel@tonic-gate 	timespec_t tslocal;
23220Sstevel@tonic-gate 	int error;
23230Sstevel@tonic-gate 
23240Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23250Sstevel@tonic-gate 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
23266812Sraf 	error = mutex_lock_impl((mutex_t *)mp, &tslocal);
23270Sstevel@tonic-gate 	if (error == ETIME)
23280Sstevel@tonic-gate 		error = ETIMEDOUT;
23290Sstevel@tonic-gate 	return (error);
23300Sstevel@tonic-gate }
23310Sstevel@tonic-gate 
23320Sstevel@tonic-gate int
pthread_mutex_reltimedlock_np(pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD reltime)23336812Sraf pthread_mutex_reltimedlock_np(pthread_mutex_t *_RESTRICT_KYWD mp,
23346812Sraf 	const struct timespec *_RESTRICT_KYWD reltime)
23350Sstevel@tonic-gate {
23360Sstevel@tonic-gate 	timespec_t tslocal;
23370Sstevel@tonic-gate 	int error;
23380Sstevel@tonic-gate 
23390Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23400Sstevel@tonic-gate 	tslocal = *reltime;
23416812Sraf 	error = mutex_lock_impl((mutex_t *)mp, &tslocal);
23420Sstevel@tonic-gate 	if (error == ETIME)
23430Sstevel@tonic-gate 		error = ETIMEDOUT;
23440Sstevel@tonic-gate 	return (error);
23450Sstevel@tonic-gate }
23460Sstevel@tonic-gate 
23476812Sraf #pragma weak pthread_mutex_trylock = mutex_trylock
23480Sstevel@tonic-gate int
mutex_trylock(mutex_t * mp)23496812Sraf mutex_trylock(mutex_t *mp)
23500Sstevel@tonic-gate {
23510Sstevel@tonic-gate 	ulwp_t *self = curthread;
23520Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
23536247Sraf 	int mtype = mp->mutex_type;
23540Sstevel@tonic-gate 	uberflags_t *gflags;
23550Sstevel@tonic-gate 
23560Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23576247Sraf 
23580Sstevel@tonic-gate 	/*
23590Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
23600Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
23610Sstevel@tonic-gate 	 * no error detection, no lock statistics,
23620Sstevel@tonic-gate 	 * and the process has only a single thread.
23630Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
23640Sstevel@tonic-gate 	 */
23656247Sraf 	if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
23660Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
23670Sstevel@tonic-gate 		/*
23680Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
23697907SRoger.Faulkner@Sun.COM 		 * We do, however, need to protect against signals.
23700Sstevel@tonic-gate 		 */
23710Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
23727907SRoger.Faulkner@Sun.COM 			sigoff(self);
23730Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
23740Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
23757907SRoger.Faulkner@Sun.COM 			sigon(self);
23760Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
23770Sstevel@tonic-gate 			return (0);
23780Sstevel@tonic-gate 		}
23794574Sraf 		if (mtype && MUTEX_OWNER(mp) == self)
23804574Sraf 			return (mutex_recursion(mp, mtype, MUTEX_TRY));
23810Sstevel@tonic-gate 		return (EBUSY);
23820Sstevel@tonic-gate 	}
23830Sstevel@tonic-gate 
23840Sstevel@tonic-gate 	/*
23850Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
23860Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
23870Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
23880Sstevel@tonic-gate 	 */
23890Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
23900Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
23910Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
23920Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
23930Sstevel@tonic-gate 			return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
23947907SRoger.Faulkner@Sun.COM 		sigoff(self);
23950Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
23960Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
23977907SRoger.Faulkner@Sun.COM 			sigon(self);
23980Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
23990Sstevel@tonic-gate 			return (0);
24000Sstevel@tonic-gate 		}
24017907SRoger.Faulkner@Sun.COM 		sigon(self);
24024574Sraf 		if (mtype && MUTEX_OWNER(mp) == self)
24034574Sraf 			return (mutex_recursion(mp, mtype, MUTEX_TRY));
24044613Sraf 		if (__td_event_report(self, TD_LOCK_TRY, udp)) {
24054613Sraf 			self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
24064613Sraf 			tdb_event(TD_LOCK_TRY, udp);
24070Sstevel@tonic-gate 		}
24084613Sraf 		return (EBUSY);
24090Sstevel@tonic-gate 	}
24100Sstevel@tonic-gate 
24110Sstevel@tonic-gate 	/* else do it the long way */
24120Sstevel@tonic-gate 	return (mutex_lock_internal(mp, NULL, MUTEX_TRY));
24130Sstevel@tonic-gate }
24140Sstevel@tonic-gate 
24150Sstevel@tonic-gate int
mutex_unlock_internal(mutex_t * mp,int retain_robust_flags)24164574Sraf mutex_unlock_internal(mutex_t *mp, int retain_robust_flags)
24170Sstevel@tonic-gate {
24180Sstevel@tonic-gate 	ulwp_t *self = curthread;
24190Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
24200Sstevel@tonic-gate 	int mtype = mp->mutex_type;
24210Sstevel@tonic-gate 	tdb_mutex_stats_t *msp;
24224574Sraf 	int error = 0;
24234574Sraf 	int release_all;
24240Sstevel@tonic-gate 	lwpid_t lwpid;
24250Sstevel@tonic-gate 
24268036SRoger.Faulkner@Sun.COM 	if ((mtype & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
24278036SRoger.Faulkner@Sun.COM 	    !mutex_held(mp))
24280Sstevel@tonic-gate 		return (EPERM);
24290Sstevel@tonic-gate 
24306812Sraf 	if (self->ul_error_detection && !mutex_held(mp))
24310Sstevel@tonic-gate 		lock_error(mp, "mutex_unlock", NULL, NULL);
24320Sstevel@tonic-gate 
24330Sstevel@tonic-gate 	if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
24340Sstevel@tonic-gate 		mp->mutex_rcount--;
24350Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
24360Sstevel@tonic-gate 		return (0);
24370Sstevel@tonic-gate 	}
24380Sstevel@tonic-gate 
24390Sstevel@tonic-gate 	if ((msp = MUTEX_STATS(mp, udp)) != NULL)
24400Sstevel@tonic-gate 		(void) record_hold_time(msp);
24410Sstevel@tonic-gate 
24424574Sraf 	if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) &&
24434574Sraf 	    (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
24448036SRoger.Faulkner@Sun.COM 		ASSERT(mtype & LOCK_ROBUST);
24454574Sraf 		mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
24464574Sraf 		mp->mutex_flag |= LOCK_NOTRECOVERABLE;
24474574Sraf 	}
24484574Sraf 	release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
24494574Sraf 
24504574Sraf 	if (mtype & LOCK_PRIO_INHERIT) {
24510Sstevel@tonic-gate 		no_preempt(self);
24520Sstevel@tonic-gate 		mp->mutex_owner = 0;
24536057Sraf 		/* mp->mutex_ownerpid is cleared by ___lwp_mutex_unlock() */
24540Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
24554574Sraf 		mp->mutex_lockw = LOCKCLEAR;
24566247Sraf 		self->ul_pilocks--;
24574574Sraf 		error = ___lwp_mutex_unlock(mp);
24580Sstevel@tonic-gate 		preempt(self);
24590Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
24605629Sraf 		mutex_unlock_process(mp, release_all);
24610Sstevel@tonic-gate 	} else {	/* USYNC_THREAD */
24624574Sraf 		if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) {
24630Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
24640Sstevel@tonic-gate 			preempt(self);
24650Sstevel@tonic-gate 		}
24660Sstevel@tonic-gate 	}
24670Sstevel@tonic-gate 
24684574Sraf 	if (mtype & LOCK_ROBUST)
24694574Sraf 		forget_lock(mp);
24704574Sraf 
24714574Sraf 	if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
24724574Sraf 		_ceil_prio_waive();
24734574Sraf 
24740Sstevel@tonic-gate 	return (error);
24750Sstevel@tonic-gate }
24760Sstevel@tonic-gate 
24776812Sraf #pragma weak pthread_mutex_unlock = mutex_unlock
24786812Sraf #pragma weak _mutex_unlock = mutex_unlock
24790Sstevel@tonic-gate int
mutex_unlock(mutex_t * mp)24806812Sraf mutex_unlock(mutex_t *mp)
24810Sstevel@tonic-gate {
24820Sstevel@tonic-gate 	ulwp_t *self = curthread;
24836247Sraf 	int mtype = mp->mutex_type;
24840Sstevel@tonic-gate 	uberflags_t *gflags;
24850Sstevel@tonic-gate 	lwpid_t lwpid;
24860Sstevel@tonic-gate 	short el;
24870Sstevel@tonic-gate 
24880Sstevel@tonic-gate 	/*
24890Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
24900Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
24910Sstevel@tonic-gate 	 * no error detection, no lock statistics,
24920Sstevel@tonic-gate 	 * and the process has only a single thread.
24930Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
24940Sstevel@tonic-gate 	 */
24956247Sraf 	if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
24966247Sraf 	    self->ul_uberdata->uberflags.uf_all) == 0) {
24970Sstevel@tonic-gate 		if (mtype) {
24980Sstevel@tonic-gate 			/*
24990Sstevel@tonic-gate 			 * At this point we know that one or both of the
25000Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
25010Sstevel@tonic-gate 			 */
25020Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
25030Sstevel@tonic-gate 				return (EPERM);
25040Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
25050Sstevel@tonic-gate 				mp->mutex_rcount--;
25060Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
25070Sstevel@tonic-gate 				return (0);
25080Sstevel@tonic-gate 			}
25090Sstevel@tonic-gate 		}
25100Sstevel@tonic-gate 		/*
25110Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
25120Sstevel@tonic-gate 		 * Also, there can be no waiters.
25130Sstevel@tonic-gate 		 */
25147907SRoger.Faulkner@Sun.COM 		sigoff(self);
25150Sstevel@tonic-gate 		mp->mutex_owner = 0;
25160Sstevel@tonic-gate 		mp->mutex_lockword = 0;
25177907SRoger.Faulkner@Sun.COM 		sigon(self);
25180Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
25190Sstevel@tonic-gate 		return (0);
25200Sstevel@tonic-gate 	}
25210Sstevel@tonic-gate 
25220Sstevel@tonic-gate 	/*
25230Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
25240Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
25250Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
25260Sstevel@tonic-gate 	 */
25270Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL) {
25280Sstevel@tonic-gate 		if (((el = gflags->uf_trs_ted) | mtype) == 0) {
25290Sstevel@tonic-gate fast_unlock:
25305629Sraf 			if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
25310Sstevel@tonic-gate 				(void) __lwp_unpark(lwpid);
25320Sstevel@tonic-gate 				preempt(self);
25330Sstevel@tonic-gate 			}
25340Sstevel@tonic-gate 			return (0);
25350Sstevel@tonic-gate 		}
25360Sstevel@tonic-gate 		if (el)		/* error detection or lock statistics */
25370Sstevel@tonic-gate 			goto slow_unlock;
25380Sstevel@tonic-gate 		if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
25390Sstevel@tonic-gate 			/*
25400Sstevel@tonic-gate 			 * At this point we know that one or both of the
25410Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
25420Sstevel@tonic-gate 			 */
25430Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
25440Sstevel@tonic-gate 				return (EPERM);
25450Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
25460Sstevel@tonic-gate 				mp->mutex_rcount--;
25470Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
25480Sstevel@tonic-gate 				return (0);
25490Sstevel@tonic-gate 			}
25500Sstevel@tonic-gate 			goto fast_unlock;
25510Sstevel@tonic-gate 		}
25520Sstevel@tonic-gate 		if ((mtype &
25530Sstevel@tonic-gate 		    ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
25540Sstevel@tonic-gate 			/*
25550Sstevel@tonic-gate 			 * At this point we know that zero, one, or both of the
25560Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and
25570Sstevel@tonic-gate 			 * that the USYNC_PROCESS flag is set.
25580Sstevel@tonic-gate 			 */
25590Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp))
25600Sstevel@tonic-gate 				return (EPERM);
25610Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
25620Sstevel@tonic-gate 				mp->mutex_rcount--;
25630Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
25640Sstevel@tonic-gate 				return (0);
25650Sstevel@tonic-gate 			}
25665629Sraf 			mutex_unlock_process(mp, 0);
25670Sstevel@tonic-gate 			return (0);
25680Sstevel@tonic-gate 		}
25690Sstevel@tonic-gate 	}
25700Sstevel@tonic-gate 
25710Sstevel@tonic-gate 	/* else do it the long way */
25720Sstevel@tonic-gate slow_unlock:
25734574Sraf 	return (mutex_unlock_internal(mp, 0));
25740Sstevel@tonic-gate }
25750Sstevel@tonic-gate 
25760Sstevel@tonic-gate /*
25770Sstevel@tonic-gate  * Internally to the library, almost all mutex lock/unlock actions
25780Sstevel@tonic-gate  * go through these lmutex_ functions, to protect critical regions.
25796812Sraf  * We replicate a bit of code from mutex_lock() and mutex_unlock()
25800Sstevel@tonic-gate  * to make these functions faster since we know that the mutex type
25810Sstevel@tonic-gate  * of all internal locks is USYNC_THREAD.  We also know that internal
25820Sstevel@tonic-gate  * locking can never fail, so we panic if it does.
25830Sstevel@tonic-gate  */
25840Sstevel@tonic-gate void
lmutex_lock(mutex_t * mp)25850Sstevel@tonic-gate lmutex_lock(mutex_t *mp)
25860Sstevel@tonic-gate {
25870Sstevel@tonic-gate 	ulwp_t *self = curthread;
25880Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
25890Sstevel@tonic-gate 
25900Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
25910Sstevel@tonic-gate 
25920Sstevel@tonic-gate 	enter_critical(self);
25930Sstevel@tonic-gate 	/*
25940Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
25950Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
25960Sstevel@tonic-gate 	 */
25970Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
25980Sstevel@tonic-gate 		/*
25990Sstevel@tonic-gate 		 * Only one thread exists; the mutex must be free.
26000Sstevel@tonic-gate 		 */
26010Sstevel@tonic-gate 		ASSERT(mp->mutex_lockw == 0);
26020Sstevel@tonic-gate 		mp->mutex_lockw = LOCKSET;
26030Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
26040Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
26050Sstevel@tonic-gate 	} else {
26060Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
26070Sstevel@tonic-gate 
26080Sstevel@tonic-gate 		if (!self->ul_schedctl_called)
26090Sstevel@tonic-gate 			(void) setup_schedctl();
26100Sstevel@tonic-gate 
26110Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
26120Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
26130Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
26144613Sraf 		} else if (mutex_trylock_adaptive(mp, 1) != 0) {
26150Sstevel@tonic-gate 			(void) mutex_lock_queue(self, msp, mp, NULL);
26160Sstevel@tonic-gate 		}
26170Sstevel@tonic-gate 
26180Sstevel@tonic-gate 		if (msp)
26190Sstevel@tonic-gate 			record_begin_hold(msp);
26200Sstevel@tonic-gate 	}
26210Sstevel@tonic-gate }
26220Sstevel@tonic-gate 
26230Sstevel@tonic-gate void
lmutex_unlock(mutex_t * mp)26240Sstevel@tonic-gate lmutex_unlock(mutex_t *mp)
26250Sstevel@tonic-gate {
26260Sstevel@tonic-gate 	ulwp_t *self = curthread;
26270Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
26280Sstevel@tonic-gate 
26290Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
26300Sstevel@tonic-gate 
26310Sstevel@tonic-gate 	/*
26320Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
26330Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
26340Sstevel@tonic-gate 	 */
26350Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
26360Sstevel@tonic-gate 		/*
26370Sstevel@tonic-gate 		 * Only one thread exists so there can be no waiters.
26380Sstevel@tonic-gate 		 */
26390Sstevel@tonic-gate 		mp->mutex_owner = 0;
26400Sstevel@tonic-gate 		mp->mutex_lockword = 0;
26410Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
26420Sstevel@tonic-gate 	} else {
26430Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
26440Sstevel@tonic-gate 		lwpid_t lwpid;
26450Sstevel@tonic-gate 
26460Sstevel@tonic-gate 		if (msp)
26470Sstevel@tonic-gate 			(void) record_hold_time(msp);
26484574Sraf 		if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
26490Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
26500Sstevel@tonic-gate 			preempt(self);
26510Sstevel@tonic-gate 		}
26520Sstevel@tonic-gate 	}
26530Sstevel@tonic-gate 	exit_critical(self);
26540Sstevel@tonic-gate }
26550Sstevel@tonic-gate 
26562248Sraf /*
26572248Sraf  * For specialized code in libc, like the asynchronous i/o code,
26582248Sraf  * the following sig_*() locking primitives are used in order
26592248Sraf  * to make the code asynchronous signal safe.  Signals are
26602248Sraf  * deferred while locks acquired by these functions are held.
26612248Sraf  */
26622248Sraf void
sig_mutex_lock(mutex_t * mp)26632248Sraf sig_mutex_lock(mutex_t *mp)
26642248Sraf {
266510637SRoger.Faulkner@Sun.COM 	ulwp_t *self = curthread;
266610637SRoger.Faulkner@Sun.COM 
266710637SRoger.Faulkner@Sun.COM 	sigoff(self);
26686515Sraf 	(void) mutex_lock(mp);
26692248Sraf }
26702248Sraf 
26712248Sraf void
sig_mutex_unlock(mutex_t * mp)26722248Sraf sig_mutex_unlock(mutex_t *mp)
26732248Sraf {
267410637SRoger.Faulkner@Sun.COM 	ulwp_t *self = curthread;
267510637SRoger.Faulkner@Sun.COM 
26766515Sraf 	(void) mutex_unlock(mp);
267710637SRoger.Faulkner@Sun.COM 	sigon(self);
26782248Sraf }
26792248Sraf 
26802248Sraf int
sig_mutex_trylock(mutex_t * mp)26812248Sraf sig_mutex_trylock(mutex_t *mp)
26822248Sraf {
268310637SRoger.Faulkner@Sun.COM 	ulwp_t *self = curthread;
26842248Sraf 	int error;
26852248Sraf 
268610637SRoger.Faulkner@Sun.COM 	sigoff(self);
26876515Sraf 	if ((error = mutex_trylock(mp)) != 0)
268810637SRoger.Faulkner@Sun.COM 		sigon(self);
26892248Sraf 	return (error);
26902248Sraf }
26912248Sraf 
26922248Sraf /*
26932248Sraf  * sig_cond_wait() is a cancellation point.
26942248Sraf  */
26952248Sraf int
sig_cond_wait(cond_t * cv,mutex_t * mp)26962248Sraf sig_cond_wait(cond_t *cv, mutex_t *mp)
26972248Sraf {
26982248Sraf 	int error;
26992248Sraf 
27002248Sraf 	ASSERT(curthread->ul_sigdefer != 0);
27016515Sraf 	pthread_testcancel();
27025891Sraf 	error = __cond_wait(cv, mp);
27032248Sraf 	if (error == EINTR && curthread->ul_cursig) {
27042248Sraf 		sig_mutex_unlock(mp);
27052248Sraf 		/* take the deferred signal here */
27062248Sraf 		sig_mutex_lock(mp);
27072248Sraf 	}
27086515Sraf 	pthread_testcancel();
27092248Sraf 	return (error);
27102248Sraf }
27112248Sraf 
27122248Sraf /*
27132248Sraf  * sig_cond_reltimedwait() is a cancellation point.
27142248Sraf  */
27152248Sraf int
sig_cond_reltimedwait(cond_t * cv,mutex_t * mp,const timespec_t * ts)27162248Sraf sig_cond_reltimedwait(cond_t *cv, mutex_t *mp, const timespec_t *ts)
27172248Sraf {
27182248Sraf 	int error;
27192248Sraf 
27202248Sraf 	ASSERT(curthread->ul_sigdefer != 0);
27216515Sraf 	pthread_testcancel();
27225891Sraf 	error = __cond_reltimedwait(cv, mp, ts);
27232248Sraf 	if (error == EINTR && curthread->ul_cursig) {
27242248Sraf 		sig_mutex_unlock(mp);
27252248Sraf 		/* take the deferred signal here */
27262248Sraf 		sig_mutex_lock(mp);
27272248Sraf 	}
27286515Sraf 	pthread_testcancel();
27292248Sraf 	return (error);
27302248Sraf }
27312248Sraf 
27325891Sraf /*
27335891Sraf  * For specialized code in libc, like the stdio code.
27345891Sraf  * the following cancel_safe_*() locking primitives are used in
27355891Sraf  * order to make the code cancellation-safe.  Cancellation is
27365891Sraf  * deferred while locks acquired by these functions are held.
27375891Sraf  */
27385891Sraf void
cancel_safe_mutex_lock(mutex_t * mp)27395891Sraf cancel_safe_mutex_lock(mutex_t *mp)
27405891Sraf {
27416515Sraf 	(void) mutex_lock(mp);
27425891Sraf 	curthread->ul_libc_locks++;
27435891Sraf }
27445891Sraf 
27455891Sraf int
cancel_safe_mutex_trylock(mutex_t * mp)27465891Sraf cancel_safe_mutex_trylock(mutex_t *mp)
27475891Sraf {
27485891Sraf 	int error;
27495891Sraf 
27506515Sraf 	if ((error = mutex_trylock(mp)) == 0)
27515891Sraf 		curthread->ul_libc_locks++;
27525891Sraf 	return (error);
27535891Sraf }
27545891Sraf 
27555891Sraf void
cancel_safe_mutex_unlock(mutex_t * mp)27565891Sraf cancel_safe_mutex_unlock(mutex_t *mp)
27575891Sraf {
27585891Sraf 	ulwp_t *self = curthread;
27595891Sraf 
27605891Sraf 	ASSERT(self->ul_libc_locks != 0);
27615891Sraf 
27626515Sraf 	(void) mutex_unlock(mp);
27635891Sraf 
27645891Sraf 	/*
27655891Sraf 	 * Decrement the count of locks held by cancel_safe_mutex_lock().
27665891Sraf 	 * If we are then in a position to terminate cleanly and
27675891Sraf 	 * if there is a pending cancellation and cancellation
27685891Sraf 	 * is not disabled and we received EINTR from a recent
27695891Sraf 	 * system call then perform the cancellation action now.
27705891Sraf 	 */
27715891Sraf 	if (--self->ul_libc_locks == 0 &&
27725891Sraf 	    !(self->ul_vfork | self->ul_nocancel |
27735891Sraf 	    self->ul_critical | self->ul_sigdefer) &&
27745891Sraf 	    cancel_active())
27756812Sraf 		pthread_exit(PTHREAD_CANCELED);
27765891Sraf }
27775891Sraf 
27780Sstevel@tonic-gate static int
shared_mutex_held(mutex_t * mparg)27790Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg)
27800Sstevel@tonic-gate {
27810Sstevel@tonic-gate 	/*
27824574Sraf 	 * The 'volatile' is necessary to make sure the compiler doesn't
27834574Sraf 	 * reorder the tests of the various components of the mutex.
27844574Sraf 	 * They must be tested in this order:
27854574Sraf 	 *	mutex_lockw
27864574Sraf 	 *	mutex_owner
27874574Sraf 	 *	mutex_ownerpid
27884574Sraf 	 * This relies on the fact that everywhere mutex_lockw is cleared,
27894574Sraf 	 * mutex_owner and mutex_ownerpid are cleared before mutex_lockw
27904574Sraf 	 * is cleared, and that everywhere mutex_lockw is set, mutex_owner
27914574Sraf 	 * and mutex_ownerpid are set after mutex_lockw is set, and that
27924574Sraf 	 * mutex_lockw is set or cleared with a memory barrier.
27930Sstevel@tonic-gate 	 */
27940Sstevel@tonic-gate 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
27950Sstevel@tonic-gate 	ulwp_t *self = curthread;
27960Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
27970Sstevel@tonic-gate 
27984574Sraf 	return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid);
27990Sstevel@tonic-gate }
28000Sstevel@tonic-gate 
28016812Sraf #pragma weak _mutex_held = mutex_held
28020Sstevel@tonic-gate int
mutex_held(mutex_t * mparg)28036812Sraf mutex_held(mutex_t *mparg)
28040Sstevel@tonic-gate {
28054574Sraf 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
28064574Sraf 
28074574Sraf 	if (mparg->mutex_type & USYNC_PROCESS)
28084574Sraf 		return (shared_mutex_held(mparg));
28090Sstevel@tonic-gate 	return (MUTEX_OWNED(mp, curthread));
28100Sstevel@tonic-gate }
28110Sstevel@tonic-gate 
28126812Sraf #pragma weak pthread_mutex_destroy = mutex_destroy
28136812Sraf #pragma weak _mutex_destroy = mutex_destroy
28140Sstevel@tonic-gate int
mutex_destroy(mutex_t * mp)28156812Sraf mutex_destroy(mutex_t *mp)
28160Sstevel@tonic-gate {
28174574Sraf 	if (mp->mutex_type & USYNC_PROCESS)
28184574Sraf 		forget_lock(mp);
28196515Sraf 	(void) memset(mp, 0, sizeof (*mp));
28200Sstevel@tonic-gate 	tdb_sync_obj_deregister(mp);
28210Sstevel@tonic-gate 	return (0);
28220Sstevel@tonic-gate }
28230Sstevel@tonic-gate 
28246812Sraf #pragma weak pthread_mutex_consistent_np = mutex_consistent
28258036SRoger.Faulkner@Sun.COM #pragma weak pthread_mutex_consistent = mutex_consistent
28264574Sraf int
mutex_consistent(mutex_t * mp)28276812Sraf mutex_consistent(mutex_t *mp)
28284574Sraf {
28294574Sraf 	/*
28304574Sraf 	 * Do this only for an inconsistent, initialized robust lock
28314574Sraf 	 * that we hold.  For all other cases, return EINVAL.
28324574Sraf 	 */
28336812Sraf 	if (mutex_held(mp) &&
28344574Sraf 	    (mp->mutex_type & LOCK_ROBUST) &&
28354574Sraf 	    (mp->mutex_flag & LOCK_INITED) &&
28364574Sraf 	    (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
28374574Sraf 		mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
28384574Sraf 		mp->mutex_rcount = 0;
28394574Sraf 		return (0);
28404574Sraf 	}
28414574Sraf 	return (EINVAL);
28424574Sraf }
28434574Sraf 
28440Sstevel@tonic-gate /*
28450Sstevel@tonic-gate  * Spin locks are separate from ordinary mutexes,
28460Sstevel@tonic-gate  * but we use the same data structure for them.
28470Sstevel@tonic-gate  */
28480Sstevel@tonic-gate 
28490Sstevel@tonic-gate int
pthread_spin_init(pthread_spinlock_t * lock,int pshared)28506812Sraf pthread_spin_init(pthread_spinlock_t *lock, int pshared)
28510Sstevel@tonic-gate {
28520Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
28530Sstevel@tonic-gate 
28546515Sraf 	(void) memset(mp, 0, sizeof (*mp));
28550Sstevel@tonic-gate 	if (pshared == PTHREAD_PROCESS_SHARED)
28560Sstevel@tonic-gate 		mp->mutex_type = USYNC_PROCESS;
28570Sstevel@tonic-gate 	else
28580Sstevel@tonic-gate 		mp->mutex_type = USYNC_THREAD;
28590Sstevel@tonic-gate 	mp->mutex_flag = LOCK_INITED;
28600Sstevel@tonic-gate 	mp->mutex_magic = MUTEX_MAGIC;
28617255Sraf 
28627255Sraf 	/*
28637255Sraf 	 * This should be at the beginning of the function,
28647255Sraf 	 * but for the sake of old broken applications that
28657255Sraf 	 * do not have proper alignment for their mutexes
28667255Sraf 	 * (and don't check the return code from pthread_spin_init),
28677255Sraf 	 * we put it here, after initializing the mutex regardless.
28687255Sraf 	 */
28697255Sraf 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
28707255Sraf 	    curthread->ul_misaligned == 0)
28717255Sraf 		return (EINVAL);
28727255Sraf 
28730Sstevel@tonic-gate 	return (0);
28740Sstevel@tonic-gate }
28750Sstevel@tonic-gate 
28760Sstevel@tonic-gate int
pthread_spin_destroy(pthread_spinlock_t * lock)28776812Sraf pthread_spin_destroy(pthread_spinlock_t *lock)
28780Sstevel@tonic-gate {
28796515Sraf 	(void) memset(lock, 0, sizeof (*lock));
28800Sstevel@tonic-gate 	return (0);
28810Sstevel@tonic-gate }
28820Sstevel@tonic-gate 
28830Sstevel@tonic-gate int
pthread_spin_trylock(pthread_spinlock_t * lock)28846812Sraf pthread_spin_trylock(pthread_spinlock_t *lock)
28850Sstevel@tonic-gate {
28860Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
28870Sstevel@tonic-gate 	ulwp_t *self = curthread;
28880Sstevel@tonic-gate 	int error = 0;
28890Sstevel@tonic-gate 
28900Sstevel@tonic-gate 	no_preempt(self);
28910Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) != 0)
28920Sstevel@tonic-gate 		error = EBUSY;
28930Sstevel@tonic-gate 	else {
28940Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
28950Sstevel@tonic-gate 		if (mp->mutex_type == USYNC_PROCESS)
28960Sstevel@tonic-gate 			mp->mutex_ownerpid = self->ul_uberdata->pid;
28970Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
28980Sstevel@tonic-gate 	}
28990Sstevel@tonic-gate 	preempt(self);
29000Sstevel@tonic-gate 	return (error);
29010Sstevel@tonic-gate }
29020Sstevel@tonic-gate 
29030Sstevel@tonic-gate int
pthread_spin_lock(pthread_spinlock_t * lock)29046812Sraf pthread_spin_lock(pthread_spinlock_t *lock)
29050Sstevel@tonic-gate {
29064574Sraf 	mutex_t *mp = (mutex_t *)lock;
29074574Sraf 	ulwp_t *self = curthread;
29084574Sraf 	volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
29094574Sraf 	int count = 0;
29104574Sraf 
29114574Sraf 	ASSERT(!self->ul_critical || self->ul_bindflags);
29124574Sraf 
29134574Sraf 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
29144574Sraf 
29150Sstevel@tonic-gate 	/*
29160Sstevel@tonic-gate 	 * We don't care whether the owner is running on a processor.
29170Sstevel@tonic-gate 	 * We just spin because that's what this interface requires.
29180Sstevel@tonic-gate 	 */
29190Sstevel@tonic-gate 	for (;;) {
29200Sstevel@tonic-gate 		if (*lockp == 0) {	/* lock byte appears to be clear */
29214574Sraf 			no_preempt(self);
29224574Sraf 			if (set_lock_byte(lockp) == 0)
29234574Sraf 				break;
29244574Sraf 			preempt(self);
29250Sstevel@tonic-gate 		}
29265629Sraf 		if (count < INT_MAX)
29275629Sraf 			count++;
29280Sstevel@tonic-gate 		SMT_PAUSE();
29290Sstevel@tonic-gate 	}
29304574Sraf 	mp->mutex_owner = (uintptr_t)self;
29314574Sraf 	if (mp->mutex_type == USYNC_PROCESS)
29324574Sraf 		mp->mutex_ownerpid = self->ul_uberdata->pid;
29334574Sraf 	preempt(self);
29345629Sraf 	if (count) {
29359397SJonathan.Haslam@Sun.COM 		DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
29365629Sraf 	}
29374574Sraf 	DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
29384574Sraf 	return (0);
29390Sstevel@tonic-gate }
29400Sstevel@tonic-gate 
29410Sstevel@tonic-gate int
pthread_spin_unlock(pthread_spinlock_t * lock)29426812Sraf pthread_spin_unlock(pthread_spinlock_t *lock)
29430Sstevel@tonic-gate {
29440Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
29450Sstevel@tonic-gate 	ulwp_t *self = curthread;
29460Sstevel@tonic-gate 
29470Sstevel@tonic-gate 	no_preempt(self);
29480Sstevel@tonic-gate 	mp->mutex_owner = 0;
29490Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
29500Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
29514570Sraf 	(void) atomic_swap_32(&mp->mutex_lockword, 0);
29520Sstevel@tonic-gate 	preempt(self);
29530Sstevel@tonic-gate 	return (0);
29540Sstevel@tonic-gate }
29550Sstevel@tonic-gate 
29565629Sraf #define	INITIAL_LOCKS	8	/* initial size of ul_heldlocks.array */
29574574Sraf 
29584574Sraf /*
29594574Sraf  * Find/allocate an entry for 'lock' in our array of held locks.
29604574Sraf  */
29614574Sraf static mutex_t **
find_lock_entry(mutex_t * lock)29624574Sraf find_lock_entry(mutex_t *lock)
29634574Sraf {
29644574Sraf 	ulwp_t *self = curthread;
29654574Sraf 	mutex_t **remembered = NULL;
29664574Sraf 	mutex_t **lockptr;
29674574Sraf 	uint_t nlocks;
29684574Sraf 
29694574Sraf 	if ((nlocks = self->ul_heldlockcnt) != 0)
29704574Sraf 		lockptr = self->ul_heldlocks.array;
29714574Sraf 	else {
29724574Sraf 		nlocks = 1;
29734574Sraf 		lockptr = &self->ul_heldlocks.single;
29744574Sraf 	}
29754574Sraf 
29764574Sraf 	for (; nlocks; nlocks--, lockptr++) {
29774574Sraf 		if (*lockptr == lock)
29784574Sraf 			return (lockptr);
29794574Sraf 		if (*lockptr == NULL && remembered == NULL)
29804574Sraf 			remembered = lockptr;
29814574Sraf 	}
29824574Sraf 	if (remembered != NULL) {
29834574Sraf 		*remembered = lock;
29844574Sraf 		return (remembered);
29854574Sraf 	}
29864574Sraf 
29874574Sraf 	/*
29884574Sraf 	 * No entry available.  Allocate more space, converting
29894574Sraf 	 * the single entry into an array of entries if necessary.
29904574Sraf 	 */
29914574Sraf 	if ((nlocks = self->ul_heldlockcnt) == 0) {
29924574Sraf 		/*
29934574Sraf 		 * Initial allocation of the array.
29944574Sraf 		 * Convert the single entry into an array.
29954574Sraf 		 */
29964574Sraf 		self->ul_heldlockcnt = nlocks = INITIAL_LOCKS;
29974574Sraf 		lockptr = lmalloc(nlocks * sizeof (mutex_t *));
29984574Sraf 		/*
29994574Sraf 		 * The single entry becomes the first entry in the array.
30004574Sraf 		 */
30014574Sraf 		*lockptr = self->ul_heldlocks.single;
30024574Sraf 		self->ul_heldlocks.array = lockptr;
30034574Sraf 		/*
30044574Sraf 		 * Return the next available entry in the array.
30054574Sraf 		 */
30064574Sraf 		*++lockptr = lock;
30074574Sraf 		return (lockptr);
30084574Sraf 	}
30094574Sraf 	/*
30104574Sraf 	 * Reallocate the array, double the size each time.
30114574Sraf 	 */
30124574Sraf 	lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *));
30136515Sraf 	(void) memcpy(lockptr, self->ul_heldlocks.array,
30144574Sraf 	    nlocks * sizeof (mutex_t *));
30154574Sraf 	lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
30164574Sraf 	self->ul_heldlocks.array = lockptr;
30174574Sraf 	self->ul_heldlockcnt *= 2;
30184574Sraf 	/*
30194574Sraf 	 * Return the next available entry in the newly allocated array.
30204574Sraf 	 */
30214574Sraf 	*(lockptr += nlocks) = lock;
30224574Sraf 	return (lockptr);
30234574Sraf }
30244574Sraf 
30254574Sraf /*
30264574Sraf  * Insert 'lock' into our list of held locks.
30274574Sraf  * Currently only used for LOCK_ROBUST mutexes.
30284574Sraf  */
30294574Sraf void
remember_lock(mutex_t * lock)30304574Sraf remember_lock(mutex_t *lock)
30314574Sraf {
30324574Sraf 	(void) find_lock_entry(lock);
30334574Sraf }
30344574Sraf 
30354574Sraf /*
30364574Sraf  * Remove 'lock' from our list of held locks.
30374574Sraf  * Currently only used for LOCK_ROBUST mutexes.
30384574Sraf  */
30394574Sraf void
forget_lock(mutex_t * lock)30404574Sraf forget_lock(mutex_t *lock)
30414574Sraf {
30424574Sraf 	*find_lock_entry(lock) = NULL;
30434574Sraf }
30444574Sraf 
30454574Sraf /*
30464574Sraf  * Free the array of held locks.
30474574Sraf  */
30484574Sraf void
heldlock_free(ulwp_t * ulwp)30494574Sraf heldlock_free(ulwp_t *ulwp)
30504574Sraf {
30514574Sraf 	uint_t nlocks;
30524574Sraf 
30534574Sraf 	if ((nlocks = ulwp->ul_heldlockcnt) != 0)
30544574Sraf 		lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
30554574Sraf 	ulwp->ul_heldlockcnt = 0;
30564574Sraf 	ulwp->ul_heldlocks.array = NULL;
30574574Sraf }
30584574Sraf 
30594574Sraf /*
30604574Sraf  * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD.
30614574Sraf  * Called from _thrp_exit() to deal with abandoned locks.
30624574Sraf  */
30634574Sraf void
heldlock_exit(void)30644574Sraf heldlock_exit(void)
30654574Sraf {
30664574Sraf 	ulwp_t *self = curthread;
30674574Sraf 	mutex_t **lockptr;
30684574Sraf 	uint_t nlocks;
30694574Sraf 	mutex_t *mp;
30704574Sraf 
30714574Sraf 	if ((nlocks = self->ul_heldlockcnt) != 0)
30724574Sraf 		lockptr = self->ul_heldlocks.array;
30734574Sraf 	else {
30744574Sraf 		nlocks = 1;
30754574Sraf 		lockptr = &self->ul_heldlocks.single;
30764574Sraf 	}
30774574Sraf 
30784574Sraf 	for (; nlocks; nlocks--, lockptr++) {
30794574Sraf 		/*
30804574Sraf 		 * The kernel takes care of transitioning held
30814574Sraf 		 * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD.
30824574Sraf 		 * We avoid that case here.
30834574Sraf 		 */
30844574Sraf 		if ((mp = *lockptr) != NULL &&
30856812Sraf 		    mutex_held(mp) &&
30864574Sraf 		    (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) ==
30874574Sraf 		    LOCK_ROBUST) {
30884574Sraf 			mp->mutex_rcount = 0;
30894574Sraf 			if (!(mp->mutex_flag & LOCK_UNMAPPED))
30904574Sraf 				mp->mutex_flag |= LOCK_OWNERDEAD;
30914574Sraf 			(void) mutex_unlock_internal(mp, 1);
30924574Sraf 		}
30934574Sraf 	}
30944574Sraf 
30954574Sraf 	heldlock_free(self);
30964574Sraf }
30974574Sraf 
30986812Sraf #pragma weak _cond_init = cond_init
30990Sstevel@tonic-gate /* ARGSUSED2 */
31000Sstevel@tonic-gate int
cond_init(cond_t * cvp,int type,void * arg)31016812Sraf cond_init(cond_t *cvp, int type, void *arg)
31020Sstevel@tonic-gate {
31030Sstevel@tonic-gate 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
31040Sstevel@tonic-gate 		return (EINVAL);
31056515Sraf 	(void) memset(cvp, 0, sizeof (*cvp));
31060Sstevel@tonic-gate 	cvp->cond_type = (uint16_t)type;
31070Sstevel@tonic-gate 	cvp->cond_magic = COND_MAGIC;
31087255Sraf 
31097255Sraf 	/*
31107255Sraf 	 * This should be at the beginning of the function,
31117255Sraf 	 * but for the sake of old broken applications that
31127255Sraf 	 * do not have proper alignment for their condvars
31137255Sraf 	 * (and don't check the return code from cond_init),
31147255Sraf 	 * we put it here, after initializing the condvar regardless.
31157255Sraf 	 */
31167255Sraf 	if (((uintptr_t)cvp & (_LONG_LONG_ALIGNMENT - 1)) &&
31177255Sraf 	    curthread->ul_misaligned == 0)
31187255Sraf 		return (EINVAL);
31197255Sraf 
31200Sstevel@tonic-gate 	return (0);
31210Sstevel@tonic-gate }
31220Sstevel@tonic-gate 
31230Sstevel@tonic-gate /*
31240Sstevel@tonic-gate  * cond_sleep_queue(): utility function for cond_wait_queue().
31250Sstevel@tonic-gate  *
31260Sstevel@tonic-gate  * Go to sleep on a condvar sleep queue, expect to be waked up
31270Sstevel@tonic-gate  * by someone calling cond_signal() or cond_broadcast() or due
31280Sstevel@tonic-gate  * to receiving a UNIX signal or being cancelled, or just simply
31290Sstevel@tonic-gate  * due to a spurious wakeup (like someome calling forkall()).
31300Sstevel@tonic-gate  *
31310Sstevel@tonic-gate  * The associated mutex is *not* reacquired before returning.
31320Sstevel@tonic-gate  * That must be done by the caller of cond_sleep_queue().
31330Sstevel@tonic-gate  */
31344574Sraf static int
cond_sleep_queue(cond_t * cvp,mutex_t * mp,timespec_t * tsp)31350Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
31360Sstevel@tonic-gate {
31370Sstevel@tonic-gate 	ulwp_t *self = curthread;
31380Sstevel@tonic-gate 	queue_head_t *qp;
31390Sstevel@tonic-gate 	queue_head_t *mqp;
31400Sstevel@tonic-gate 	lwpid_t lwpid;
31410Sstevel@tonic-gate 	int signalled;
31420Sstevel@tonic-gate 	int error;
31436247Sraf 	int cv_wake;
31444574Sraf 	int release_all;
31450Sstevel@tonic-gate 
31460Sstevel@tonic-gate 	/*
31470Sstevel@tonic-gate 	 * Put ourself on the CV sleep queue, unlock the mutex, then
31480Sstevel@tonic-gate 	 * park ourself and unpark a candidate lwp to grab the mutex.
31490Sstevel@tonic-gate 	 * We must go onto the CV sleep queue before dropping the
31500Sstevel@tonic-gate 	 * mutex in order to guarantee atomicity of the operation.
31510Sstevel@tonic-gate 	 */
31520Sstevel@tonic-gate 	self->ul_sp = stkptr();
31530Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
31546247Sraf 	enqueue(qp, self, 0);
31550Sstevel@tonic-gate 	cvp->cond_waiters_user = 1;
31560Sstevel@tonic-gate 	self->ul_cvmutex = mp;
31576247Sraf 	self->ul_cv_wake = cv_wake = (tsp != NULL);
31580Sstevel@tonic-gate 	self->ul_signalled = 0;
31594574Sraf 	if (mp->mutex_flag & LOCK_OWNERDEAD) {
31604574Sraf 		mp->mutex_flag &= ~LOCK_OWNERDEAD;
31614574Sraf 		mp->mutex_flag |= LOCK_NOTRECOVERABLE;
31624574Sraf 	}
31634574Sraf 	release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
31644574Sraf 	lwpid = mutex_unlock_queue(mp, release_all);
31650Sstevel@tonic-gate 	for (;;) {
31660Sstevel@tonic-gate 		set_parking_flag(self, 1);
31670Sstevel@tonic-gate 		queue_unlock(qp);
31680Sstevel@tonic-gate 		if (lwpid != 0) {
31690Sstevel@tonic-gate 			lwpid = preempt_unpark(self, lwpid);
31700Sstevel@tonic-gate 			preempt(self);
31710Sstevel@tonic-gate 		}
31720Sstevel@tonic-gate 		/*
31730Sstevel@tonic-gate 		 * We may have a deferred signal present,
31740Sstevel@tonic-gate 		 * in which case we should return EINTR.
31750Sstevel@tonic-gate 		 * Also, we may have received a SIGCANCEL; if so
31760Sstevel@tonic-gate 		 * and we are cancelable we should return EINTR.
31770Sstevel@tonic-gate 		 * We force an immediate EINTR return from
31780Sstevel@tonic-gate 		 * __lwp_park() by turning our parking flag off.
31790Sstevel@tonic-gate 		 */
31800Sstevel@tonic-gate 		if (self->ul_cursig != 0 ||
31810Sstevel@tonic-gate 		    (self->ul_cancelable && self->ul_cancel_pending))
31820Sstevel@tonic-gate 			set_parking_flag(self, 0);
31830Sstevel@tonic-gate 		/*
31840Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
31850Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
31860Sstevel@tonic-gate 		 */
31870Sstevel@tonic-gate 		error = __lwp_park(tsp, lwpid);
31880Sstevel@tonic-gate 		set_parking_flag(self, 0);
31890Sstevel@tonic-gate 		lwpid = 0;	/* unpark the other lwp only once */
31900Sstevel@tonic-gate 		/*
31910Sstevel@tonic-gate 		 * We were waked up by cond_signal(), cond_broadcast(),
31920Sstevel@tonic-gate 		 * by an interrupt or timeout (EINTR or ETIME),
31930Sstevel@tonic-gate 		 * or we may just have gotten a spurious wakeup.
31940Sstevel@tonic-gate 		 */
31950Sstevel@tonic-gate 		qp = queue_lock(cvp, CV);
31966247Sraf 		if (!cv_wake)
31976247Sraf 			mqp = queue_lock(mp, MX);
31980Sstevel@tonic-gate 		if (self->ul_sleepq == NULL)
31990Sstevel@tonic-gate 			break;
32000Sstevel@tonic-gate 		/*
32010Sstevel@tonic-gate 		 * We are on either the condvar sleep queue or the
32021893Sraf 		 * mutex sleep queue.  Break out of the sleep if we
32031893Sraf 		 * were interrupted or we timed out (EINTR or ETIME).
32040Sstevel@tonic-gate 		 * Else this is a spurious wakeup; continue the loop.
32050Sstevel@tonic-gate 		 */
32066247Sraf 		if (!cv_wake && self->ul_sleepq == mqp) { /* mutex queue */
32071893Sraf 			if (error) {
32086247Sraf 				mp->mutex_waiters = dequeue_self(mqp);
32091893Sraf 				break;
32101893Sraf 			}
32111893Sraf 			tsp = NULL;	/* no more timeout */
32121893Sraf 		} else if (self->ul_sleepq == qp) {	/* condvar queue */
32130Sstevel@tonic-gate 			if (error) {
32146247Sraf 				cvp->cond_waiters_user = dequeue_self(qp);
32150Sstevel@tonic-gate 				break;
32160Sstevel@tonic-gate 			}
32170Sstevel@tonic-gate 			/*
32180Sstevel@tonic-gate 			 * Else a spurious wakeup on the condvar queue.
32190Sstevel@tonic-gate 			 * __lwp_park() has already adjusted the timeout.
32200Sstevel@tonic-gate 			 */
32210Sstevel@tonic-gate 		} else {
32220Sstevel@tonic-gate 			thr_panic("cond_sleep_queue(): thread not on queue");
32230Sstevel@tonic-gate 		}
32246247Sraf 		if (!cv_wake)
32256247Sraf 			queue_unlock(mqp);
32260Sstevel@tonic-gate 	}
32270Sstevel@tonic-gate 
32280Sstevel@tonic-gate 	self->ul_sp = 0;
32296247Sraf 	self->ul_cv_wake = 0;
32306247Sraf 	ASSERT(self->ul_cvmutex == NULL);
32310Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
32320Sstevel@tonic-gate 	    self->ul_wchan == NULL);
32330Sstevel@tonic-gate 
32340Sstevel@tonic-gate 	signalled = self->ul_signalled;
32350Sstevel@tonic-gate 	self->ul_signalled = 0;
32360Sstevel@tonic-gate 	queue_unlock(qp);
32376247Sraf 	if (!cv_wake)
32386247Sraf 		queue_unlock(mqp);
32390Sstevel@tonic-gate 
32400Sstevel@tonic-gate 	/*
32410Sstevel@tonic-gate 	 * If we were concurrently cond_signal()d and any of:
32420Sstevel@tonic-gate 	 * received a UNIX signal, were cancelled, or got a timeout,
32430Sstevel@tonic-gate 	 * then perform another cond_signal() to avoid consuming it.
32440Sstevel@tonic-gate 	 */
32450Sstevel@tonic-gate 	if (error && signalled)
32466812Sraf 		(void) cond_signal(cvp);
32470Sstevel@tonic-gate 
32480Sstevel@tonic-gate 	return (error);
32490Sstevel@tonic-gate }
32500Sstevel@tonic-gate 
32517255Sraf static void
cond_wait_check_alignment(cond_t * cvp,mutex_t * mp)32527255Sraf cond_wait_check_alignment(cond_t *cvp, mutex_t *mp)
32537255Sraf {
32547255Sraf 	if ((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1))
32557255Sraf 		lock_error(mp, "cond_wait", cvp, "mutex is misaligned");
32567255Sraf 	if ((uintptr_t)cvp & (_LONG_LONG_ALIGNMENT - 1))
32577255Sraf 		lock_error(mp, "cond_wait", cvp, "condvar is misaligned");
32587255Sraf }
32597255Sraf 
32600Sstevel@tonic-gate int
cond_wait_queue(cond_t * cvp,mutex_t * mp,timespec_t * tsp)32615629Sraf cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
32620Sstevel@tonic-gate {
32630Sstevel@tonic-gate 	ulwp_t *self = curthread;
32640Sstevel@tonic-gate 	int error;
32654574Sraf 	int merror;
32660Sstevel@tonic-gate 
32677255Sraf 	if (self->ul_error_detection && self->ul_misaligned == 0)
32687255Sraf 		cond_wait_check_alignment(cvp, mp);
32697255Sraf 
32700Sstevel@tonic-gate 	/*
32710Sstevel@tonic-gate 	 * The old thread library was programmed to defer signals
32720Sstevel@tonic-gate 	 * while in cond_wait() so that the associated mutex would
32730Sstevel@tonic-gate 	 * be guaranteed to be held when the application signal
32740Sstevel@tonic-gate 	 * handler was invoked.
32750Sstevel@tonic-gate 	 *
32760Sstevel@tonic-gate 	 * We do not behave this way by default; the state of the
32770Sstevel@tonic-gate 	 * associated mutex in the signal handler is undefined.
32780Sstevel@tonic-gate 	 *
32790Sstevel@tonic-gate 	 * To accommodate applications that depend on the old
32800Sstevel@tonic-gate 	 * behavior, the _THREAD_COND_WAIT_DEFER environment
32810Sstevel@tonic-gate 	 * variable can be set to 1 and we will behave in the
32820Sstevel@tonic-gate 	 * old way with respect to cond_wait().
32830Sstevel@tonic-gate 	 */
32840Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
32850Sstevel@tonic-gate 		sigoff(self);
32860Sstevel@tonic-gate 
32870Sstevel@tonic-gate 	error = cond_sleep_queue(cvp, mp, tsp);
32880Sstevel@tonic-gate 
32890Sstevel@tonic-gate 	/*
32900Sstevel@tonic-gate 	 * Reacquire the mutex.
32910Sstevel@tonic-gate 	 */
32925629Sraf 	if ((merror = mutex_lock_impl(mp, NULL)) != 0)
32934574Sraf 		error = merror;
32940Sstevel@tonic-gate 
32950Sstevel@tonic-gate 	/*
32960Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
32970Sstevel@tonic-gate 	 */
32980Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
32990Sstevel@tonic-gate 		sigon(self);
33000Sstevel@tonic-gate 
33010Sstevel@tonic-gate 	return (error);
33020Sstevel@tonic-gate }
33030Sstevel@tonic-gate 
33040Sstevel@tonic-gate /*
33050Sstevel@tonic-gate  * cond_sleep_kernel(): utility function for cond_wait_kernel().
33060Sstevel@tonic-gate  * See the comment ahead of cond_sleep_queue(), above.
33070Sstevel@tonic-gate  */
33084574Sraf static int
cond_sleep_kernel(cond_t * cvp,mutex_t * mp,timespec_t * tsp)33090Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
33100Sstevel@tonic-gate {
33110Sstevel@tonic-gate 	int mtype = mp->mutex_type;
33120Sstevel@tonic-gate 	ulwp_t *self = curthread;
33130Sstevel@tonic-gate 	int error;
33140Sstevel@tonic-gate 
33154574Sraf 	if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
33164574Sraf 		_ceil_prio_waive();
33170Sstevel@tonic-gate 
33180Sstevel@tonic-gate 	self->ul_sp = stkptr();
33190Sstevel@tonic-gate 	self->ul_wchan = cvp;
33207907SRoger.Faulkner@Sun.COM 	sigoff(self);
33210Sstevel@tonic-gate 	mp->mutex_owner = 0;
33226057Sraf 	/* mp->mutex_ownerpid is cleared by ___lwp_cond_wait() */
33236247Sraf 	if (mtype & LOCK_PRIO_INHERIT) {
33240Sstevel@tonic-gate 		mp->mutex_lockw = LOCKCLEAR;
33256247Sraf 		self->ul_pilocks--;
33266247Sraf 	}
33270Sstevel@tonic-gate 	/*
33280Sstevel@tonic-gate 	 * ___lwp_cond_wait() returns immediately with EINTR if
33290Sstevel@tonic-gate 	 * set_parking_flag(self,0) is called on this lwp before it
33300Sstevel@tonic-gate 	 * goes to sleep in the kernel.  sigacthandler() calls this
33310Sstevel@tonic-gate 	 * when a deferred signal is noted.  This assures that we don't
33320Sstevel@tonic-gate 	 * get stuck in ___lwp_cond_wait() with all signals blocked
33330Sstevel@tonic-gate 	 * due to taking a deferred signal before going to sleep.
33340Sstevel@tonic-gate 	 */
33350Sstevel@tonic-gate 	set_parking_flag(self, 1);
33360Sstevel@tonic-gate 	if (self->ul_cursig != 0 ||
33370Sstevel@tonic-gate 	    (self->ul_cancelable && self->ul_cancel_pending))
33380Sstevel@tonic-gate 		set_parking_flag(self, 0);
33390Sstevel@tonic-gate 	error = ___lwp_cond_wait(cvp, mp, tsp, 1);
33400Sstevel@tonic-gate 	set_parking_flag(self, 0);
33417907SRoger.Faulkner@Sun.COM 	sigon(self);
33420Sstevel@tonic-gate 	self->ul_sp = 0;
33430Sstevel@tonic-gate 	self->ul_wchan = NULL;
33440Sstevel@tonic-gate 	return (error);
33450Sstevel@tonic-gate }
33460Sstevel@tonic-gate 
33470Sstevel@tonic-gate int
cond_wait_kernel(cond_t * cvp,mutex_t * mp,timespec_t * tsp)33480Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
33490Sstevel@tonic-gate {
33500Sstevel@tonic-gate 	ulwp_t *self = curthread;
33510Sstevel@tonic-gate 	int error;
33520Sstevel@tonic-gate 	int merror;
33530Sstevel@tonic-gate 
33547255Sraf 	if (self->ul_error_detection && self->ul_misaligned == 0)
33557255Sraf 		cond_wait_check_alignment(cvp, mp);
33567255Sraf 
33570Sstevel@tonic-gate 	/*
33580Sstevel@tonic-gate 	 * See the large comment in cond_wait_queue(), above.
33590Sstevel@tonic-gate 	 */
33600Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
33610Sstevel@tonic-gate 		sigoff(self);
33620Sstevel@tonic-gate 
33630Sstevel@tonic-gate 	error = cond_sleep_kernel(cvp, mp, tsp);
33640Sstevel@tonic-gate 
33650Sstevel@tonic-gate 	/*
33660Sstevel@tonic-gate 	 * Override the return code from ___lwp_cond_wait()
33670Sstevel@tonic-gate 	 * with any non-zero return code from mutex_lock().
33680Sstevel@tonic-gate 	 * This addresses robust lock failures in particular;
33690Sstevel@tonic-gate 	 * the caller must see the EOWNERDEAD or ENOTRECOVERABLE
33700Sstevel@tonic-gate 	 * errors in order to take corrective action.
33710Sstevel@tonic-gate 	 */
33725629Sraf 	if ((merror = mutex_lock_impl(mp, NULL)) != 0)
33730Sstevel@tonic-gate 		error = merror;
33740Sstevel@tonic-gate 
33750Sstevel@tonic-gate 	/*
33760Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
33770Sstevel@tonic-gate 	 */
33780Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
33790Sstevel@tonic-gate 		sigon(self);
33800Sstevel@tonic-gate 
33810Sstevel@tonic-gate 	return (error);
33820Sstevel@tonic-gate }
33830Sstevel@tonic-gate 
33840Sstevel@tonic-gate /*
33856812Sraf  * Common code for cond_wait() and cond_timedwait()
33860Sstevel@tonic-gate  */
33870Sstevel@tonic-gate int
cond_wait_common(cond_t * cvp,mutex_t * mp,timespec_t * tsp)33880Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
33890Sstevel@tonic-gate {
33900Sstevel@tonic-gate 	int mtype = mp->mutex_type;
33910Sstevel@tonic-gate 	hrtime_t begin_sleep = 0;
33920Sstevel@tonic-gate 	ulwp_t *self = curthread;
33930Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
33940Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
33950Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
33960Sstevel@tonic-gate 	uint8_t rcount;
33970Sstevel@tonic-gate 	int error = 0;
33980Sstevel@tonic-gate 
33990Sstevel@tonic-gate 	/*
34000Sstevel@tonic-gate 	 * The SUSV3 Posix spec for pthread_cond_timedwait() states:
34010Sstevel@tonic-gate 	 *	Except in the case of [ETIMEDOUT], all these error checks
34020Sstevel@tonic-gate 	 *	shall act as if they were performed immediately at the
34030Sstevel@tonic-gate 	 *	beginning of processing for the function and shall cause
34040Sstevel@tonic-gate 	 *	an error return, in effect, prior to modifying the state
34050Sstevel@tonic-gate 	 *	of the mutex specified by mutex or the condition variable
34060Sstevel@tonic-gate 	 *	specified by cond.
34070Sstevel@tonic-gate 	 * Therefore, we must return EINVAL now if the timout is invalid.
34080Sstevel@tonic-gate 	 */
34090Sstevel@tonic-gate 	if (tsp != NULL &&
34100Sstevel@tonic-gate 	    (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC))
34110Sstevel@tonic-gate 		return (EINVAL);
34120Sstevel@tonic-gate 
34130Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
34140Sstevel@tonic-gate 		self->ul_sp = stkptr();
34150Sstevel@tonic-gate 		self->ul_wchan = cvp;
34160Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
34170Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = cvp;
34180Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
34190Sstevel@tonic-gate 		self->ul_sp = 0;
34200Sstevel@tonic-gate 	}
34210Sstevel@tonic-gate 	if (csp) {
34220Sstevel@tonic-gate 		if (tsp)
34230Sstevel@tonic-gate 			tdb_incr(csp->cond_timedwait);
34240Sstevel@tonic-gate 		else
34250Sstevel@tonic-gate 			tdb_incr(csp->cond_wait);
34260Sstevel@tonic-gate 	}
34270Sstevel@tonic-gate 	if (msp)
34280Sstevel@tonic-gate 		begin_sleep = record_hold_time(msp);
34290Sstevel@tonic-gate 	else if (csp)
34300Sstevel@tonic-gate 		begin_sleep = gethrtime();
34310Sstevel@tonic-gate 
34320Sstevel@tonic-gate 	if (self->ul_error_detection) {
34336812Sraf 		if (!mutex_held(mp))
34340Sstevel@tonic-gate 			lock_error(mp, "cond_wait", cvp, NULL);
34350Sstevel@tonic-gate 		if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0)
34360Sstevel@tonic-gate 			lock_error(mp, "recursive mutex in cond_wait",
34375629Sraf 			    cvp, NULL);
34380Sstevel@tonic-gate 		if (cvp->cond_type & USYNC_PROCESS) {
34394574Sraf 			if (!(mtype & USYNC_PROCESS))
34400Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
34415629Sraf 				    "condvar process-shared, "
34425629Sraf 				    "mutex process-private");
34430Sstevel@tonic-gate 		} else {
34444574Sraf 			if (mtype & USYNC_PROCESS)
34450Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
34465629Sraf 				    "condvar process-private, "
34475629Sraf 				    "mutex process-shared");
34480Sstevel@tonic-gate 		}
34490Sstevel@tonic-gate 	}
34500Sstevel@tonic-gate 
34510Sstevel@tonic-gate 	/*
34520Sstevel@tonic-gate 	 * We deal with recursive mutexes by completely
34530Sstevel@tonic-gate 	 * dropping the lock and restoring the recursion
34540Sstevel@tonic-gate 	 * count after waking up.  This is arguably wrong,
34550Sstevel@tonic-gate 	 * but it obeys the principle of least astonishment.
34560Sstevel@tonic-gate 	 */
34570Sstevel@tonic-gate 	rcount = mp->mutex_rcount;
34580Sstevel@tonic-gate 	mp->mutex_rcount = 0;
34594574Sraf 	if ((mtype &
34604574Sraf 	    (USYNC_PROCESS | LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) |
34610Sstevel@tonic-gate 	    (cvp->cond_type & USYNC_PROCESS))
34620Sstevel@tonic-gate 		error = cond_wait_kernel(cvp, mp, tsp);
34630Sstevel@tonic-gate 	else
34645629Sraf 		error = cond_wait_queue(cvp, mp, tsp);
34650Sstevel@tonic-gate 	mp->mutex_rcount = rcount;
34660Sstevel@tonic-gate 
34670Sstevel@tonic-gate 	if (csp) {
34680Sstevel@tonic-gate 		hrtime_t lapse = gethrtime() - begin_sleep;
34690Sstevel@tonic-gate 		if (tsp == NULL)
34700Sstevel@tonic-gate 			csp->cond_wait_sleep_time += lapse;
34710Sstevel@tonic-gate 		else {
34720Sstevel@tonic-gate 			csp->cond_timedwait_sleep_time += lapse;
34730Sstevel@tonic-gate 			if (error == ETIME)
34740Sstevel@tonic-gate 				tdb_incr(csp->cond_timedwait_timeout);
34750Sstevel@tonic-gate 		}
34760Sstevel@tonic-gate 	}
34770Sstevel@tonic-gate 	return (error);
34780Sstevel@tonic-gate }
34790Sstevel@tonic-gate 
34800Sstevel@tonic-gate /*
34816812Sraf  * cond_wait() is a cancellation point but __cond_wait() is not.
34826812Sraf  * Internally, libc calls the non-cancellation version.
34835891Sraf  * Other libraries need to use pthread_setcancelstate(), as appropriate,
34845891Sraf  * since __cond_wait() is not exported from libc.
34850Sstevel@tonic-gate  */
34860Sstevel@tonic-gate int
__cond_wait(cond_t * cvp,mutex_t * mp)34875891Sraf __cond_wait(cond_t *cvp, mutex_t *mp)
34880Sstevel@tonic-gate {
34890Sstevel@tonic-gate 	ulwp_t *self = curthread;
34900Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
34910Sstevel@tonic-gate 	uberflags_t *gflags;
34920Sstevel@tonic-gate 
34938036SRoger.Faulkner@Sun.COM 	if ((mp->mutex_type & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
34948036SRoger.Faulkner@Sun.COM 	    !mutex_held(mp))
34958036SRoger.Faulkner@Sun.COM 		return (EPERM);
34968036SRoger.Faulkner@Sun.COM 
34970Sstevel@tonic-gate 	/*
34980Sstevel@tonic-gate 	 * Optimize the common case of USYNC_THREAD plus
34990Sstevel@tonic-gate 	 * no error detection, no lock statistics, and no event tracing.
35000Sstevel@tonic-gate 	 */
35010Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
35020Sstevel@tonic-gate 	    (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted |
35030Sstevel@tonic-gate 	    self->ul_td_events_enable |
35040Sstevel@tonic-gate 	    udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0)
35055629Sraf 		return (cond_wait_queue(cvp, mp, NULL));
35060Sstevel@tonic-gate 
35070Sstevel@tonic-gate 	/*
35080Sstevel@tonic-gate 	 * Else do it the long way.
35090Sstevel@tonic-gate 	 */
35100Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, NULL));
35110Sstevel@tonic-gate }
35120Sstevel@tonic-gate 
35136812Sraf #pragma weak _cond_wait = cond_wait
35140Sstevel@tonic-gate int
cond_wait(cond_t * cvp,mutex_t * mp)35156812Sraf cond_wait(cond_t *cvp, mutex_t *mp)
35160Sstevel@tonic-gate {
35170Sstevel@tonic-gate 	int error;
35180Sstevel@tonic-gate 
35190Sstevel@tonic-gate 	_cancelon();
35205891Sraf 	error = __cond_wait(cvp, mp);
35210Sstevel@tonic-gate 	if (error == EINTR)
35220Sstevel@tonic-gate 		_canceloff();
35230Sstevel@tonic-gate 	else
35240Sstevel@tonic-gate 		_canceloff_nocancel();
35250Sstevel@tonic-gate 	return (error);
35260Sstevel@tonic-gate }
35270Sstevel@tonic-gate 
35285891Sraf /*
35295891Sraf  * pthread_cond_wait() is a cancellation point.
35305891Sraf  */
35310Sstevel@tonic-gate int
pthread_cond_wait(pthread_cond_t * _RESTRICT_KYWD cvp,pthread_mutex_t * _RESTRICT_KYWD mp)35326812Sraf pthread_cond_wait(pthread_cond_t *_RESTRICT_KYWD cvp,
35336812Sraf 	pthread_mutex_t *_RESTRICT_KYWD mp)
35340Sstevel@tonic-gate {
35350Sstevel@tonic-gate 	int error;
35360Sstevel@tonic-gate 
35376812Sraf 	error = cond_wait((cond_t *)cvp, (mutex_t *)mp);
35380Sstevel@tonic-gate 	return ((error == EINTR)? 0 : error);
35390Sstevel@tonic-gate }
35400Sstevel@tonic-gate 
35410Sstevel@tonic-gate /*
35426812Sraf  * cond_timedwait() is a cancellation point but __cond_timedwait() is not.
35430Sstevel@tonic-gate  */
35440Sstevel@tonic-gate int
__cond_timedwait(cond_t * cvp,mutex_t * mp,const timespec_t * abstime)35455891Sraf __cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
35460Sstevel@tonic-gate {
35470Sstevel@tonic-gate 	clockid_t clock_id = cvp->cond_clockid;
35480Sstevel@tonic-gate 	timespec_t reltime;
35490Sstevel@tonic-gate 	int error;
35500Sstevel@tonic-gate 
35518036SRoger.Faulkner@Sun.COM 	if ((mp->mutex_type & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
35528036SRoger.Faulkner@Sun.COM 	    !mutex_held(mp))
35538036SRoger.Faulkner@Sun.COM 		return (EPERM);
35548036SRoger.Faulkner@Sun.COM 
35550Sstevel@tonic-gate 	if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES)
35560Sstevel@tonic-gate 		clock_id = CLOCK_REALTIME;
35570Sstevel@tonic-gate 	abstime_to_reltime(clock_id, abstime, &reltime);
35580Sstevel@tonic-gate 	error = cond_wait_common(cvp, mp, &reltime);
35590Sstevel@tonic-gate 	if (error == ETIME && clock_id == CLOCK_HIGHRES) {
35600Sstevel@tonic-gate 		/*
35610Sstevel@tonic-gate 		 * Don't return ETIME if we didn't really get a timeout.
35620Sstevel@tonic-gate 		 * This can happen if we return because someone resets
35630Sstevel@tonic-gate 		 * the system clock.  Just return zero in this case,
35640Sstevel@tonic-gate 		 * giving a spurious wakeup but not a timeout.
35650Sstevel@tonic-gate 		 */
35660Sstevel@tonic-gate 		if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC +
35670Sstevel@tonic-gate 		    abstime->tv_nsec > gethrtime())
35680Sstevel@tonic-gate 			error = 0;
35690Sstevel@tonic-gate 	}
35700Sstevel@tonic-gate 	return (error);
35710Sstevel@tonic-gate }
35720Sstevel@tonic-gate 
35730Sstevel@tonic-gate int
cond_timedwait(cond_t * cvp,mutex_t * mp,const timespec_t * abstime)35746812Sraf cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
35750Sstevel@tonic-gate {
35760Sstevel@tonic-gate 	int error;
35770Sstevel@tonic-gate 
35780Sstevel@tonic-gate 	_cancelon();
35795891Sraf 	error = __cond_timedwait(cvp, mp, abstime);
35800Sstevel@tonic-gate 	if (error == EINTR)
35810Sstevel@tonic-gate 		_canceloff();
35820Sstevel@tonic-gate 	else
35830Sstevel@tonic-gate 		_canceloff_nocancel();
35840Sstevel@tonic-gate 	return (error);
35850Sstevel@tonic-gate }
35860Sstevel@tonic-gate 
35875891Sraf /*
35885891Sraf  * pthread_cond_timedwait() is a cancellation point.
35895891Sraf  */
35900Sstevel@tonic-gate int
pthread_cond_timedwait(pthread_cond_t * _RESTRICT_KYWD cvp,pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD abstime)35916812Sraf pthread_cond_timedwait(pthread_cond_t *_RESTRICT_KYWD cvp,
35926812Sraf 	pthread_mutex_t *_RESTRICT_KYWD mp,
35936812Sraf 	const struct timespec *_RESTRICT_KYWD abstime)
35940Sstevel@tonic-gate {
35950Sstevel@tonic-gate 	int error;
35960Sstevel@tonic-gate 
35976812Sraf 	error = cond_timedwait((cond_t *)cvp, (mutex_t *)mp, abstime);
35980Sstevel@tonic-gate 	if (error == ETIME)
35990Sstevel@tonic-gate 		error = ETIMEDOUT;
36000Sstevel@tonic-gate 	else if (error == EINTR)
36010Sstevel@tonic-gate 		error = 0;
36020Sstevel@tonic-gate 	return (error);
36030Sstevel@tonic-gate }
36040Sstevel@tonic-gate 
36050Sstevel@tonic-gate /*
36066812Sraf  * cond_reltimedwait() is a cancellation point but __cond_reltimedwait() is not.
36070Sstevel@tonic-gate  */
36080Sstevel@tonic-gate int
__cond_reltimedwait(cond_t * cvp,mutex_t * mp,const timespec_t * reltime)36095891Sraf __cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
36100Sstevel@tonic-gate {
36110Sstevel@tonic-gate 	timespec_t tslocal = *reltime;
36120Sstevel@tonic-gate 
36138036SRoger.Faulkner@Sun.COM 	if ((mp->mutex_type & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
36148036SRoger.Faulkner@Sun.COM 	    !mutex_held(mp))
36158036SRoger.Faulkner@Sun.COM 		return (EPERM);
36168036SRoger.Faulkner@Sun.COM 
36170Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, &tslocal));
36180Sstevel@tonic-gate }
36190Sstevel@tonic-gate 
36200Sstevel@tonic-gate int
cond_reltimedwait(cond_t * cvp,mutex_t * mp,const timespec_t * reltime)36216812Sraf cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
36220Sstevel@tonic-gate {
36230Sstevel@tonic-gate 	int error;
36240Sstevel@tonic-gate 
36250Sstevel@tonic-gate 	_cancelon();
36265891Sraf 	error = __cond_reltimedwait(cvp, mp, reltime);
36270Sstevel@tonic-gate 	if (error == EINTR)
36280Sstevel@tonic-gate 		_canceloff();
36290Sstevel@tonic-gate 	else
36300Sstevel@tonic-gate 		_canceloff_nocancel();
36310Sstevel@tonic-gate 	return (error);
36320Sstevel@tonic-gate }
36330Sstevel@tonic-gate 
36340Sstevel@tonic-gate int
pthread_cond_reltimedwait_np(pthread_cond_t * _RESTRICT_KYWD cvp,pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD reltime)36356812Sraf pthread_cond_reltimedwait_np(pthread_cond_t *_RESTRICT_KYWD cvp,
36366812Sraf 	pthread_mutex_t *_RESTRICT_KYWD mp,
36376812Sraf 	const struct timespec *_RESTRICT_KYWD reltime)
36380Sstevel@tonic-gate {
36390Sstevel@tonic-gate 	int error;
36400Sstevel@tonic-gate 
36416812Sraf 	error = cond_reltimedwait((cond_t *)cvp, (mutex_t *)mp, reltime);
36420Sstevel@tonic-gate 	if (error == ETIME)
36430Sstevel@tonic-gate 		error = ETIMEDOUT;
36440Sstevel@tonic-gate 	else if (error == EINTR)
36450Sstevel@tonic-gate 		error = 0;
36460Sstevel@tonic-gate 	return (error);
36470Sstevel@tonic-gate }
36480Sstevel@tonic-gate 
36496812Sraf #pragma weak pthread_cond_signal = cond_signal
36506812Sraf #pragma weak _cond_signal = cond_signal
36510Sstevel@tonic-gate int
cond_signal(cond_t * cvp)36526812Sraf cond_signal(cond_t *cvp)
36530Sstevel@tonic-gate {
36540Sstevel@tonic-gate 	ulwp_t *self = curthread;
36550Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
36560Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
36570Sstevel@tonic-gate 	int error = 0;
36586247Sraf 	int more;
36596247Sraf 	lwpid_t lwpid;
36600Sstevel@tonic-gate 	queue_head_t *qp;
36610Sstevel@tonic-gate 	mutex_t *mp;
36620Sstevel@tonic-gate 	queue_head_t *mqp;
36630Sstevel@tonic-gate 	ulwp_t **ulwpp;
36640Sstevel@tonic-gate 	ulwp_t *ulwp;
36656247Sraf 	ulwp_t *prev;
36660Sstevel@tonic-gate 
36670Sstevel@tonic-gate 	if (csp)
36680Sstevel@tonic-gate 		tdb_incr(csp->cond_signal);
36690Sstevel@tonic-gate 
36700Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
36716812Sraf 		error = _lwp_cond_signal(cvp);
36720Sstevel@tonic-gate 
36730Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
36740Sstevel@tonic-gate 		return (error);
36750Sstevel@tonic-gate 
36760Sstevel@tonic-gate 	/*
36770Sstevel@tonic-gate 	 * Move someone from the condvar sleep queue to the mutex sleep
36780Sstevel@tonic-gate 	 * queue for the mutex that he will acquire on being waked up.
36790Sstevel@tonic-gate 	 * We can do this only if we own the mutex he will acquire.
36800Sstevel@tonic-gate 	 * If we do not own the mutex, or if his ul_cv_wake flag
36810Sstevel@tonic-gate 	 * is set, just dequeue and unpark him.
36820Sstevel@tonic-gate 	 */
36830Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
36846247Sraf 	ulwpp = queue_slot(qp, &prev, &more);
36856247Sraf 	cvp->cond_waiters_user = more;
36866247Sraf 	if (ulwpp == NULL) {	/* no one on the sleep queue */
36870Sstevel@tonic-gate 		queue_unlock(qp);
36880Sstevel@tonic-gate 		return (error);
36890Sstevel@tonic-gate 	}
36906247Sraf 	ulwp = *ulwpp;
36910Sstevel@tonic-gate 
36920Sstevel@tonic-gate 	/*
36930Sstevel@tonic-gate 	 * Inform the thread that he was the recipient of a cond_signal().
36940Sstevel@tonic-gate 	 * This lets him deal with cond_signal() and, concurrently,
36950Sstevel@tonic-gate 	 * one or more of a cancellation, a UNIX signal, or a timeout.
36960Sstevel@tonic-gate 	 * These latter conditions must not consume a cond_signal().
36970Sstevel@tonic-gate 	 */
36980Sstevel@tonic-gate 	ulwp->ul_signalled = 1;
36990Sstevel@tonic-gate 
37000Sstevel@tonic-gate 	/*
37010Sstevel@tonic-gate 	 * Dequeue the waiter but leave his ul_sleepq non-NULL
37020Sstevel@tonic-gate 	 * while we move him to the mutex queue so that he can
37030Sstevel@tonic-gate 	 * deal properly with spurious wakeups.
37040Sstevel@tonic-gate 	 */
37056247Sraf 	queue_unlink(qp, ulwpp, prev);
37060Sstevel@tonic-gate 
37070Sstevel@tonic-gate 	mp = ulwp->ul_cvmutex;		/* the mutex he will acquire */
37080Sstevel@tonic-gate 	ulwp->ul_cvmutex = NULL;
37090Sstevel@tonic-gate 	ASSERT(mp != NULL);
37100Sstevel@tonic-gate 
37110Sstevel@tonic-gate 	if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
37126247Sraf 		/* just wake him up */
37136247Sraf 		lwpid = ulwp->ul_lwpid;
37140Sstevel@tonic-gate 		no_preempt(self);
37150Sstevel@tonic-gate 		ulwp->ul_sleepq = NULL;
37160Sstevel@tonic-gate 		ulwp->ul_wchan = NULL;
37170Sstevel@tonic-gate 		queue_unlock(qp);
37180Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
37190Sstevel@tonic-gate 		preempt(self);
37200Sstevel@tonic-gate 	} else {
37216247Sraf 		/* move him to the mutex queue */
37220Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
37236247Sraf 		enqueue(mqp, ulwp, 0);
37240Sstevel@tonic-gate 		mp->mutex_waiters = 1;
37250Sstevel@tonic-gate 		queue_unlock(mqp);
37260Sstevel@tonic-gate 		queue_unlock(qp);
37270Sstevel@tonic-gate 	}
37280Sstevel@tonic-gate 
37290Sstevel@tonic-gate 	return (error);
37300Sstevel@tonic-gate }
37310Sstevel@tonic-gate 
37324570Sraf /*
37334574Sraf  * Utility function called by mutex_wakeup_all(), cond_broadcast(),
37344574Sraf  * and rw_queue_release() to (re)allocate a big buffer to hold the
37354574Sraf  * lwpids of all the threads to be set running after they are removed
37364574Sraf  * from their sleep queues.  Since we are holding a queue lock, we
37374574Sraf  * cannot call any function that might acquire a lock.  mmap(), munmap(),
37384574Sraf  * lwp_unpark_all() are simple system calls and are safe in this regard.
37394570Sraf  */
37404570Sraf lwpid_t *
alloc_lwpids(lwpid_t * lwpid,int * nlwpid_ptr,int * maxlwps_ptr)37414570Sraf alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr)
37424570Sraf {
37434570Sraf 	/*
37444570Sraf 	 * Allocate NEWLWPS ids on the first overflow.
37454570Sraf 	 * Double the allocation each time after that.
37464570Sraf 	 */
37474570Sraf 	int nlwpid = *nlwpid_ptr;
37484570Sraf 	int maxlwps = *maxlwps_ptr;
37494570Sraf 	int first_allocation;
37504570Sraf 	int newlwps;
37514570Sraf 	void *vaddr;
37524570Sraf 
37534570Sraf 	ASSERT(nlwpid == maxlwps);
37544570Sraf 
37554570Sraf 	first_allocation = (maxlwps == MAXLWPS);
37564570Sraf 	newlwps = first_allocation? NEWLWPS : 2 * maxlwps;
37576515Sraf 	vaddr = mmap(NULL, newlwps * sizeof (lwpid_t),
37584570Sraf 	    PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0);
37594570Sraf 
37604570Sraf 	if (vaddr == MAP_FAILED) {
37614570Sraf 		/*
37624570Sraf 		 * Let's hope this never happens.
37634570Sraf 		 * If it does, then we have a terrible
37644570Sraf 		 * thundering herd on our hands.
37654570Sraf 		 */
37664570Sraf 		(void) __lwp_unpark_all(lwpid, nlwpid);
37674570Sraf 		*nlwpid_ptr = 0;
37684570Sraf 	} else {
37696515Sraf 		(void) memcpy(vaddr, lwpid, maxlwps * sizeof (lwpid_t));
37704570Sraf 		if (!first_allocation)
37716515Sraf 			(void) munmap((caddr_t)lwpid,
37724570Sraf 			    maxlwps * sizeof (lwpid_t));
37734570Sraf 		lwpid = vaddr;
37744570Sraf 		*maxlwps_ptr = newlwps;
37754570Sraf 	}
37764570Sraf 
37774570Sraf 	return (lwpid);
37784570Sraf }
37790Sstevel@tonic-gate 
37806812Sraf #pragma weak pthread_cond_broadcast = cond_broadcast
37816812Sraf #pragma weak _cond_broadcast = cond_broadcast
37820Sstevel@tonic-gate int
cond_broadcast(cond_t * cvp)37836812Sraf cond_broadcast(cond_t *cvp)
37840Sstevel@tonic-gate {
37850Sstevel@tonic-gate 	ulwp_t *self = curthread;
37860Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
37870Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
37880Sstevel@tonic-gate 	int error = 0;
37890Sstevel@tonic-gate 	queue_head_t *qp;
37906247Sraf 	queue_root_t *qrp;
37910Sstevel@tonic-gate 	mutex_t *mp;
37920Sstevel@tonic-gate 	mutex_t *mp_cache = NULL;
37934570Sraf 	queue_head_t *mqp = NULL;
37940Sstevel@tonic-gate 	ulwp_t *ulwp;
37954570Sraf 	int nlwpid = 0;
37964570Sraf 	int maxlwps = MAXLWPS;
37970Sstevel@tonic-gate 	lwpid_t buffer[MAXLWPS];
37980Sstevel@tonic-gate 	lwpid_t *lwpid = buffer;
37990Sstevel@tonic-gate 
38000Sstevel@tonic-gate 	if (csp)
38010Sstevel@tonic-gate 		tdb_incr(csp->cond_broadcast);
38020Sstevel@tonic-gate 
38030Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
38046812Sraf 		error = _lwp_cond_broadcast(cvp);
38050Sstevel@tonic-gate 
38060Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
38070Sstevel@tonic-gate 		return (error);
38080Sstevel@tonic-gate 
38090Sstevel@tonic-gate 	/*
38100Sstevel@tonic-gate 	 * Move everyone from the condvar sleep queue to the mutex sleep
38110Sstevel@tonic-gate 	 * queue for the mutex that they will acquire on being waked up.
38120Sstevel@tonic-gate 	 * We can do this only if we own the mutex they will acquire.
38130Sstevel@tonic-gate 	 * If we do not own the mutex, or if their ul_cv_wake flag
38140Sstevel@tonic-gate 	 * is set, just dequeue and unpark them.
38150Sstevel@tonic-gate 	 *
38160Sstevel@tonic-gate 	 * We keep track of lwpids that are to be unparked in lwpid[].
38170Sstevel@tonic-gate 	 * __lwp_unpark_all() is called to unpark all of them after
38180Sstevel@tonic-gate 	 * they have been removed from the sleep queue and the sleep
38190Sstevel@tonic-gate 	 * queue lock has been dropped.  If we run out of space in our
38200Sstevel@tonic-gate 	 * on-stack buffer, we need to allocate more but we can't call
38210Sstevel@tonic-gate 	 * lmalloc() because we are holding a queue lock when the overflow
38220Sstevel@tonic-gate 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
38234570Sraf 	 * either because the application may have allocated a small
38244570Sraf 	 * stack and we don't want to overrun the stack.  So we call
38254570Sraf 	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
38260Sstevel@tonic-gate 	 * system call directly since that path acquires no locks.
38270Sstevel@tonic-gate 	 */
38280Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
38290Sstevel@tonic-gate 	cvp->cond_waiters_user = 0;
38306247Sraf 	for (;;) {
38316247Sraf 		if ((qrp = qp->qh_root) == NULL ||
38326247Sraf 		    (ulwp = qrp->qr_head) == NULL)
38336247Sraf 			break;
38346247Sraf 		ASSERT(ulwp->ul_wchan == cvp);
38356247Sraf 		queue_unlink(qp, &qrp->qr_head, NULL);
38360Sstevel@tonic-gate 		mp = ulwp->ul_cvmutex;		/* his mutex */
38370Sstevel@tonic-gate 		ulwp->ul_cvmutex = NULL;
38380Sstevel@tonic-gate 		ASSERT(mp != NULL);
38390Sstevel@tonic-gate 		if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
38406247Sraf 			/* just wake him up */
38410Sstevel@tonic-gate 			ulwp->ul_sleepq = NULL;
38420Sstevel@tonic-gate 			ulwp->ul_wchan = NULL;
38434570Sraf 			if (nlwpid == maxlwps)
38444570Sraf 				lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
38450Sstevel@tonic-gate 			lwpid[nlwpid++] = ulwp->ul_lwpid;
38460Sstevel@tonic-gate 		} else {
38476247Sraf 			/* move him to the mutex queue */
38480Sstevel@tonic-gate 			if (mp != mp_cache) {
38490Sstevel@tonic-gate 				mp_cache = mp;
38504570Sraf 				if (mqp != NULL)
38514570Sraf 					queue_unlock(mqp);
38524570Sraf 				mqp = queue_lock(mp, MX);
38530Sstevel@tonic-gate 			}
38546247Sraf 			enqueue(mqp, ulwp, 0);
38550Sstevel@tonic-gate 			mp->mutex_waiters = 1;
38560Sstevel@tonic-gate 		}
38570Sstevel@tonic-gate 	}
38584570Sraf 	if (mqp != NULL)
38594570Sraf 		queue_unlock(mqp);
38604570Sraf 	if (nlwpid == 0) {
38614570Sraf 		queue_unlock(qp);
38624570Sraf 	} else {
38634570Sraf 		no_preempt(self);
38644570Sraf 		queue_unlock(qp);
38650Sstevel@tonic-gate 		if (nlwpid == 1)
38660Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid[0]);
38670Sstevel@tonic-gate 		else
38680Sstevel@tonic-gate 			(void) __lwp_unpark_all(lwpid, nlwpid);
38694570Sraf 		preempt(self);
38700Sstevel@tonic-gate 	}
38710Sstevel@tonic-gate 	if (lwpid != buffer)
38726515Sraf 		(void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
38730Sstevel@tonic-gate 	return (error);
38740Sstevel@tonic-gate }
38750Sstevel@tonic-gate 
38766812Sraf #pragma weak pthread_cond_destroy = cond_destroy
38770Sstevel@tonic-gate int
cond_destroy(cond_t * cvp)38786812Sraf cond_destroy(cond_t *cvp)
38790Sstevel@tonic-gate {
38800Sstevel@tonic-gate 	cvp->cond_magic = 0;
38810Sstevel@tonic-gate 	tdb_sync_obj_deregister(cvp);
38820Sstevel@tonic-gate 	return (0);
38830Sstevel@tonic-gate }
38840Sstevel@tonic-gate 
38850Sstevel@tonic-gate #if defined(THREAD_DEBUG)
38860Sstevel@tonic-gate void
assert_no_libc_locks_held(void)38870Sstevel@tonic-gate assert_no_libc_locks_held(void)
38880Sstevel@tonic-gate {
38890Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
38900Sstevel@tonic-gate }
38910Sstevel@tonic-gate 
38920Sstevel@tonic-gate /* protected by link_lock */
38930Sstevel@tonic-gate uint64_t spin_lock_spin;
38940Sstevel@tonic-gate uint64_t spin_lock_spin2;
38950Sstevel@tonic-gate uint64_t spin_lock_sleep;
38960Sstevel@tonic-gate uint64_t spin_lock_wakeup;
38970Sstevel@tonic-gate 
38980Sstevel@tonic-gate /*
38990Sstevel@tonic-gate  * Record spin lock statistics.
39000Sstevel@tonic-gate  * Called by a thread exiting itself in thrp_exit().
39010Sstevel@tonic-gate  * Also called via atexit() from the thread calling
39020Sstevel@tonic-gate  * exit() to do all the other threads as well.
39030Sstevel@tonic-gate  */
39040Sstevel@tonic-gate void
record_spin_locks(ulwp_t * ulwp)39050Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp)
39060Sstevel@tonic-gate {
39070Sstevel@tonic-gate 	spin_lock_spin += ulwp->ul_spin_lock_spin;
39080Sstevel@tonic-gate 	spin_lock_spin2 += ulwp->ul_spin_lock_spin2;
39090Sstevel@tonic-gate 	spin_lock_sleep += ulwp->ul_spin_lock_sleep;
39100Sstevel@tonic-gate 	spin_lock_wakeup += ulwp->ul_spin_lock_wakeup;
39110Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin = 0;
39120Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin2 = 0;
39130Sstevel@tonic-gate 	ulwp->ul_spin_lock_sleep = 0;
39140Sstevel@tonic-gate 	ulwp->ul_spin_lock_wakeup = 0;
39150Sstevel@tonic-gate }
39160Sstevel@tonic-gate 
39170Sstevel@tonic-gate /*
39180Sstevel@tonic-gate  * atexit function:  dump the queue statistics to stderr.
39190Sstevel@tonic-gate  */
39200Sstevel@tonic-gate #include <stdio.h>
39210Sstevel@tonic-gate void
dump_queue_statistics(void)39220Sstevel@tonic-gate dump_queue_statistics(void)
39230Sstevel@tonic-gate {
39240Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
39250Sstevel@tonic-gate 	queue_head_t *qp;
39260Sstevel@tonic-gate 	int qn;
39270Sstevel@tonic-gate 	uint64_t spin_lock_total = 0;
39280Sstevel@tonic-gate 
39290Sstevel@tonic-gate 	if (udp->queue_head == NULL || thread_queue_dump == 0)
39300Sstevel@tonic-gate 		return;
39310Sstevel@tonic-gate 
39320Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 ||
39336247Sraf 	    fprintf(stderr, "queue#   lockcount    max qlen    max hlen\n") < 0)
39340Sstevel@tonic-gate 		return;
39350Sstevel@tonic-gate 	for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) {
39360Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
39370Sstevel@tonic-gate 			continue;
39380Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
39396247Sraf 		if (fprintf(stderr, "%5d %12llu%12u%12u\n", qn,
39406247Sraf 		    (u_longlong_t)qp->qh_lockcount,
39416247Sraf 		    qp->qh_qmax, qp->qh_hmax) < 0)
39425629Sraf 			return;
39430Sstevel@tonic-gate 	}
39440Sstevel@tonic-gate 
39450Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 ||
39466247Sraf 	    fprintf(stderr, "queue#   lockcount    max qlen    max hlen\n") < 0)
39470Sstevel@tonic-gate 		return;
39480Sstevel@tonic-gate 	for (qn = 0; qn < QHASHSIZE; qn++, qp++) {
39490Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
39500Sstevel@tonic-gate 			continue;
39510Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
39526247Sraf 		if (fprintf(stderr, "%5d %12llu%12u%12u\n", qn,
39536247Sraf 		    (u_longlong_t)qp->qh_lockcount,
39546247Sraf 		    qp->qh_qmax, qp->qh_hmax) < 0)
39555629Sraf 			return;
39560Sstevel@tonic-gate 	}
39570Sstevel@tonic-gate 
39580Sstevel@tonic-gate 	(void) fprintf(stderr, "\n  spin_lock_total  = %10llu\n",
39595629Sraf 	    (u_longlong_t)spin_lock_total);
39600Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin   = %10llu\n",
39615629Sraf 	    (u_longlong_t)spin_lock_spin);
39620Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin2  = %10llu\n",
39635629Sraf 	    (u_longlong_t)spin_lock_spin2);
39640Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_sleep  = %10llu\n",
39655629Sraf 	    (u_longlong_t)spin_lock_sleep);
39660Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_wakeup = %10llu\n",
39675629Sraf 	    (u_longlong_t)spin_lock_wakeup);
39680Sstevel@tonic-gate }
39696247Sraf #endif
3970