10Sstevel@tonic-gate /*
20Sstevel@tonic-gate * CDDL HEADER START
30Sstevel@tonic-gate *
40Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51893Sraf * Common Development and Distribution License (the "License").
61893Sraf * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate *
80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate * See the License for the specific language governing permissions
110Sstevel@tonic-gate * and limitations under the License.
120Sstevel@tonic-gate *
130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate *
190Sstevel@tonic-gate * CDDL HEADER END
200Sstevel@tonic-gate */
211219Sraf
220Sstevel@tonic-gate /*
239170SRoger.Faulkner@Sun.COM * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
240Sstevel@tonic-gate * Use is subject to license terms.
250Sstevel@tonic-gate */
260Sstevel@tonic-gate
270Sstevel@tonic-gate #include "lint.h"
280Sstevel@tonic-gate #include "thr_uberdata.h"
296247Sraf #include <sys/rtpriocntl.h>
306057Sraf #include <sys/sdt.h>
316057Sraf #include <atomic.h>
320Sstevel@tonic-gate
336247Sraf #if defined(THREAD_DEBUG)
346247Sraf #define INCR32(x) (((x) != UINT32_MAX)? (x)++ : 0)
356247Sraf #define INCR(x) ((x)++)
366247Sraf #define DECR(x) ((x)--)
376247Sraf #define MAXINCR(m, x) ((m < ++x)? (m = x) : 0)
386247Sraf #else
396247Sraf #define INCR32(x)
406247Sraf #define INCR(x)
416247Sraf #define DECR(x)
426247Sraf #define MAXINCR(m, x)
436247Sraf #endif
446247Sraf
450Sstevel@tonic-gate /*
460Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1.
470Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock()
484574Sraf * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error.
490Sstevel@tonic-gate */
500Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX;
510Sstevel@tonic-gate
520Sstevel@tonic-gate static int shared_mutex_held(mutex_t *);
534574Sraf static int mutex_queuelock_adaptive(mutex_t *);
544574Sraf static void mutex_wakeup_all(mutex_t *);
550Sstevel@tonic-gate
560Sstevel@tonic-gate /*
570Sstevel@tonic-gate * Lock statistics support functions.
580Sstevel@tonic-gate */
590Sstevel@tonic-gate void
record_begin_hold(tdb_mutex_stats_t * msp)600Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp)
610Sstevel@tonic-gate {
620Sstevel@tonic-gate tdb_incr(msp->mutex_lock);
630Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime();
640Sstevel@tonic-gate }
650Sstevel@tonic-gate
660Sstevel@tonic-gate hrtime_t
record_hold_time(tdb_mutex_stats_t * msp)670Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp)
680Sstevel@tonic-gate {
690Sstevel@tonic-gate hrtime_t now = gethrtime();
700Sstevel@tonic-gate
710Sstevel@tonic-gate if (msp->mutex_begin_hold)
720Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold;
730Sstevel@tonic-gate msp->mutex_begin_hold = 0;
740Sstevel@tonic-gate return (now);
750Sstevel@tonic-gate }
760Sstevel@tonic-gate
770Sstevel@tonic-gate /*
780Sstevel@tonic-gate * Called once at library initialization.
790Sstevel@tonic-gate */
800Sstevel@tonic-gate void
mutex_setup(void)810Sstevel@tonic-gate mutex_setup(void)
820Sstevel@tonic-gate {
830Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw))
840Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex");
850Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread;
860Sstevel@tonic-gate }
870Sstevel@tonic-gate
880Sstevel@tonic-gate /*
895629Sraf * The default spin count of 1000 is experimentally determined.
905629Sraf * On sun4u machines with any number of processors it could be raised
910Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference.
925629Sraf * The environment variable:
930Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count
945629Sraf * can be used to override and set the count in the range [0 .. 1,000,000].
950Sstevel@tonic-gate */
960Sstevel@tonic-gate int thread_adaptive_spin = 1000;
970Sstevel@tonic-gate uint_t thread_max_spinners = 100;
980Sstevel@tonic-gate int thread_queue_verify = 0;
990Sstevel@tonic-gate static int ncpus;
1000Sstevel@tonic-gate
1010Sstevel@tonic-gate /*
1020Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks.
1035629Sraf * We try harder to acquire queue locks by spinning.
1040Sstevel@tonic-gate * The environment variable:
1050Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count
1060Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000].
1070Sstevel@tonic-gate */
1085629Sraf int thread_queue_spin = 10000;
1090Sstevel@tonic-gate
1104574Sraf #define ALL_ATTRIBUTES \
1114574Sraf (LOCK_RECURSIVE | LOCK_ERRORCHECK | \
1124574Sraf LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT | \
1134574Sraf LOCK_ROBUST)
1140Sstevel@tonic-gate
1150Sstevel@tonic-gate /*
1164574Sraf * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST,
1174574Sraf * augmented by zero or more the flags:
1184574Sraf * LOCK_RECURSIVE
1194574Sraf * LOCK_ERRORCHECK
1204574Sraf * LOCK_PRIO_INHERIT
1214574Sraf * LOCK_PRIO_PROTECT
1224574Sraf * LOCK_ROBUST
1230Sstevel@tonic-gate */
1246812Sraf #pragma weak _mutex_init = mutex_init
1250Sstevel@tonic-gate /* ARGSUSED2 */
1260Sstevel@tonic-gate int
mutex_init(mutex_t * mp,int type,void * arg)1276812Sraf mutex_init(mutex_t *mp, int type, void *arg)
1280Sstevel@tonic-gate {
1294574Sraf int basetype = (type & ~ALL_ATTRIBUTES);
1306247Sraf const pcclass_t *pccp;
1314574Sraf int error = 0;
1326247Sraf int ceil;
1334574Sraf
1344574Sraf if (basetype == USYNC_PROCESS_ROBUST) {
1354574Sraf /*
1364574Sraf * USYNC_PROCESS_ROBUST is a deprecated historical type.
1374574Sraf * We change it into (USYNC_PROCESS | LOCK_ROBUST) but
1384574Sraf * retain the USYNC_PROCESS_ROBUST flag so we can return
1394574Sraf * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST
1404574Sraf * mutexes will ever draw ELOCKUNMAPPED).
1414574Sraf */
1424574Sraf type |= (USYNC_PROCESS | LOCK_ROBUST);
1434574Sraf basetype = USYNC_PROCESS;
1444574Sraf }
1454574Sraf
1466247Sraf if (type & LOCK_PRIO_PROTECT)
1476247Sraf pccp = get_info_by_policy(SCHED_FIFO);
1486247Sraf if ((basetype != USYNC_THREAD && basetype != USYNC_PROCESS) ||
1494574Sraf (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT))
1506247Sraf == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT) ||
1516247Sraf ((type & LOCK_PRIO_PROTECT) &&
1526247Sraf ((ceil = *(int *)arg) < pccp->pcc_primin ||
1536247Sraf ceil > pccp->pcc_primax))) {
1544574Sraf error = EINVAL;
1554574Sraf } else if (type & LOCK_ROBUST) {
1564574Sraf /*
1574574Sraf * Callers of mutex_init() with the LOCK_ROBUST attribute
1584574Sraf * are required to pass an initially all-zero mutex.
1594574Sraf * Multiple calls to mutex_init() are allowed; all but
1604574Sraf * the first return EBUSY. A call to mutex_init() is
1614574Sraf * allowed to make an inconsistent robust lock consistent
1624574Sraf * (for historical usage, even though the proper interface
1634574Sraf * for this is mutex_consistent()). Note that we use
1644574Sraf * atomic_or_16() to set the LOCK_INITED flag so as
1654574Sraf * not to disturb surrounding bits (LOCK_OWNERDEAD, etc).
1664574Sraf */
1674574Sraf if (!(mp->mutex_flag & LOCK_INITED)) {
1684574Sraf mp->mutex_type = (uint8_t)type;
1696812Sraf atomic_or_16(&mp->mutex_flag, LOCK_INITED);
1704574Sraf mp->mutex_magic = MUTEX_MAGIC;
1714574Sraf } else if (type != mp->mutex_type ||
1726247Sraf ((type & LOCK_PRIO_PROTECT) && mp->mutex_ceiling != ceil)) {
1734574Sraf error = EINVAL;
1746812Sraf } else if (mutex_consistent(mp) != 0) {
1754574Sraf error = EBUSY;
1764574Sraf }
1774574Sraf /* register a process robust mutex with the kernel */
1784574Sraf if (basetype == USYNC_PROCESS)
1794574Sraf register_lock(mp);
1804574Sraf } else {
1816515Sraf (void) memset(mp, 0, sizeof (*mp));
1820Sstevel@tonic-gate mp->mutex_type = (uint8_t)type;
1830Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED;
1844574Sraf mp->mutex_magic = MUTEX_MAGIC;
1850Sstevel@tonic-gate }
1864574Sraf
1876247Sraf if (error == 0 && (type & LOCK_PRIO_PROTECT)) {
1886247Sraf mp->mutex_ceiling = ceil;
1896247Sraf }
1904574Sraf
1917255Sraf /*
1927255Sraf * This should be at the beginning of the function,
1937255Sraf * but for the sake of old broken applications that
1947255Sraf * do not have proper alignment for their mutexes
1957255Sraf * (and don't check the return code from mutex_init),
1967255Sraf * we put it here, after initializing the mutex regardless.
1977255Sraf */
1987255Sraf if (error == 0 &&
1997255Sraf ((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
2007255Sraf curthread->ul_misaligned == 0)
2017255Sraf error = EINVAL;
2027255Sraf
2030Sstevel@tonic-gate return (error);
2040Sstevel@tonic-gate }
2050Sstevel@tonic-gate
2060Sstevel@tonic-gate /*
2076247Sraf * Delete mp from list of ceiling mutexes owned by curthread.
2080Sstevel@tonic-gate * Return 1 if the head of the chain was updated.
2090Sstevel@tonic-gate */
2100Sstevel@tonic-gate int
_ceil_mylist_del(mutex_t * mp)2110Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp)
2120Sstevel@tonic-gate {
2130Sstevel@tonic-gate ulwp_t *self = curthread;
2140Sstevel@tonic-gate mxchain_t **mcpp;
2150Sstevel@tonic-gate mxchain_t *mcp;
2160Sstevel@tonic-gate
2176247Sraf for (mcpp = &self->ul_mxchain;
2186247Sraf (mcp = *mcpp) != NULL;
2196247Sraf mcpp = &mcp->mxchain_next) {
2206247Sraf if (mcp->mxchain_mx == mp) {
2216247Sraf *mcpp = mcp->mxchain_next;
2226247Sraf lfree(mcp, sizeof (*mcp));
2236247Sraf return (mcpp == &self->ul_mxchain);
2246247Sraf }
2256247Sraf }
2266247Sraf return (0);
2270Sstevel@tonic-gate }
2280Sstevel@tonic-gate
2290Sstevel@tonic-gate /*
2306247Sraf * Add mp to the list of ceiling mutexes owned by curthread.
2310Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated.
2320Sstevel@tonic-gate */
2330Sstevel@tonic-gate int
_ceil_mylist_add(mutex_t * mp)2340Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp)
2350Sstevel@tonic-gate {
2360Sstevel@tonic-gate ulwp_t *self = curthread;
2370Sstevel@tonic-gate mxchain_t *mcp;
2380Sstevel@tonic-gate
2390Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL)
2400Sstevel@tonic-gate return (ENOMEM);
2410Sstevel@tonic-gate mcp->mxchain_mx = mp;
2420Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain;
2430Sstevel@tonic-gate self->ul_mxchain = mcp;
2440Sstevel@tonic-gate return (0);
2450Sstevel@tonic-gate }
2460Sstevel@tonic-gate
2470Sstevel@tonic-gate /*
2486247Sraf * Helper function for _ceil_prio_inherit() and _ceil_prio_waive(), below.
2496247Sraf */
2506247Sraf static void
set_rt_priority(ulwp_t * self,int prio)2516247Sraf set_rt_priority(ulwp_t *self, int prio)
2526247Sraf {
2536247Sraf pcparms_t pcparm;
2546247Sraf
2556247Sraf pcparm.pc_cid = self->ul_rtclassid;
2566247Sraf ((rtparms_t *)pcparm.pc_clparms)->rt_tqnsecs = RT_NOCHANGE;
2576247Sraf ((rtparms_t *)pcparm.pc_clparms)->rt_pri = prio;
2586515Sraf (void) priocntl(P_LWPID, self->ul_lwpid, PC_SETPARMS, &pcparm);
2596247Sraf }
2606247Sraf
2616247Sraf /*
2626247Sraf * Inherit priority from ceiling.
2636247Sraf * This changes the effective priority, not the assigned priority.
2640Sstevel@tonic-gate */
2650Sstevel@tonic-gate void
_ceil_prio_inherit(int prio)2666247Sraf _ceil_prio_inherit(int prio)
2670Sstevel@tonic-gate {
2680Sstevel@tonic-gate ulwp_t *self = curthread;
2696247Sraf
2706247Sraf self->ul_epri = prio;
2716247Sraf set_rt_priority(self, prio);
2720Sstevel@tonic-gate }
2730Sstevel@tonic-gate
2740Sstevel@tonic-gate /*
2750Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks
2760Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this
2770Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority.
2780Sstevel@tonic-gate */
2790Sstevel@tonic-gate void
_ceil_prio_waive(void)2800Sstevel@tonic-gate _ceil_prio_waive(void)
2810Sstevel@tonic-gate {
2820Sstevel@tonic-gate ulwp_t *self = curthread;
2836247Sraf mxchain_t *mcp = self->ul_mxchain;
2846247Sraf int prio;
2856247Sraf
2866247Sraf if (mcp == NULL) {
2876247Sraf prio = self->ul_pri;
2886247Sraf self->ul_epri = 0;
2890Sstevel@tonic-gate } else {
2906247Sraf prio = mcp->mxchain_mx->mutex_ceiling;
2916247Sraf self->ul_epri = prio;
2920Sstevel@tonic-gate }
2936247Sraf set_rt_priority(self, prio);
2940Sstevel@tonic-gate }
2950Sstevel@tonic-gate
2960Sstevel@tonic-gate /*
2975629Sraf * Clear the lock byte. Retain the waiters byte and the spinners byte.
2985629Sraf * Return the old value of the lock word.
2995629Sraf */
3005629Sraf static uint32_t
clear_lockbyte(volatile uint32_t * lockword)3015629Sraf clear_lockbyte(volatile uint32_t *lockword)
3025629Sraf {
3035629Sraf uint32_t old;
3045629Sraf uint32_t new;
3055629Sraf
3065629Sraf do {
3075629Sraf old = *lockword;
3085629Sraf new = old & ~LOCKMASK;
3095629Sraf } while (atomic_cas_32(lockword, old, new) != old);
3105629Sraf
3115629Sraf return (old);
3125629Sraf }
3135629Sraf
3145629Sraf /*
3156057Sraf * Same as clear_lockbyte(), but operates on mutex_lockword64.
3166057Sraf * The mutex_ownerpid field is cleared along with the lock byte.
3176057Sraf */
3186057Sraf static uint64_t
clear_lockbyte64(volatile uint64_t * lockword64)3196057Sraf clear_lockbyte64(volatile uint64_t *lockword64)
3206057Sraf {
3216057Sraf uint64_t old;
3226057Sraf uint64_t new;
3236057Sraf
3246057Sraf do {
3256057Sraf old = *lockword64;
3266057Sraf new = old & ~LOCKMASK64;
3276057Sraf } while (atomic_cas_64(lockword64, old, new) != old);
3286057Sraf
3296057Sraf return (old);
3306057Sraf }
3316057Sraf
3326057Sraf /*
3336057Sraf * Similar to set_lock_byte(), which only tries to set the lock byte.
3347255Sraf * Here, we attempt to set the lock byte AND the mutex_ownerpid, keeping
3357255Sraf * the remaining bytes constant. This atomic operation is required for the
3367255Sraf * correctness of process-shared robust locks, otherwise there would be
3377255Sraf * a window or vulnerability in which the lock byte had been set but the
3387255Sraf * mutex_ownerpid had not yet been set. If the process were to die in
3397255Sraf * this window of vulnerability (due to some other thread calling exit()
3407255Sraf * or the process receiving a fatal signal), the mutex would be left locked
3417255Sraf * but without a process-ID to determine which process was holding the lock.
3427255Sraf * The kernel would then be unable to mark the robust mutex as LOCK_OWNERDEAD
3437255Sraf * when the process died. For all other cases of process-shared locks, this
3447255Sraf * operation is just a convenience, for the sake of common code.
3457255Sraf *
3467255Sraf * This operation requires process-shared robust locks to be properly
3477255Sraf * aligned on an 8-byte boundary, at least on sparc machines, lest the
3487255Sraf * operation incur an alignment fault. This is automatic when locks
3497255Sraf * are declared properly using the mutex_t or pthread_mutex_t data types
3507255Sraf * and the application does not allocate dynamic memory on less than an
3517255Sraf * 8-byte boundary. See the 'horrible hack' comments below for cases
3527255Sraf * dealing with such broken applications.
3536057Sraf */
3546057Sraf static int
set_lock_byte64(volatile uint64_t * lockword64,pid_t ownerpid)3556057Sraf set_lock_byte64(volatile uint64_t *lockword64, pid_t ownerpid)
3566057Sraf {
3576057Sraf uint64_t old;
3586057Sraf uint64_t new;
3596057Sraf
3606057Sraf old = *lockword64 & ~LOCKMASK64;
3616057Sraf new = old | ((uint64_t)(uint_t)ownerpid << PIDSHIFT) | LOCKBYTE64;
3626057Sraf if (atomic_cas_64(lockword64, old, new) == old)
3636057Sraf return (LOCKCLEAR);
3646057Sraf
3656057Sraf return (LOCKSET);
3666057Sraf }
3676057Sraf
3686057Sraf /*
3695629Sraf * Increment the spinners count in the mutex lock word.
3705629Sraf * Return 0 on success. Return -1 if the count would overflow.
3715629Sraf */
3725629Sraf static int
spinners_incr(volatile uint32_t * lockword,uint8_t max_spinners)3735629Sraf spinners_incr(volatile uint32_t *lockword, uint8_t max_spinners)
3745629Sraf {
3755629Sraf uint32_t old;
3765629Sraf uint32_t new;
3775629Sraf
3785629Sraf do {
3795629Sraf old = *lockword;
3805629Sraf if (((old & SPINNERMASK) >> SPINNERSHIFT) >= max_spinners)
3815629Sraf return (-1);
3825629Sraf new = old + (1 << SPINNERSHIFT);
3835629Sraf } while (atomic_cas_32(lockword, old, new) != old);
3845629Sraf
3855629Sraf return (0);
3865629Sraf }
3875629Sraf
3885629Sraf /*
3895629Sraf * Decrement the spinners count in the mutex lock word.
3905629Sraf * Return the new value of the lock word.
3915629Sraf */
3925629Sraf static uint32_t
spinners_decr(volatile uint32_t * lockword)3935629Sraf spinners_decr(volatile uint32_t *lockword)
3945629Sraf {
3955629Sraf uint32_t old;
3965629Sraf uint32_t new;
3975629Sraf
3985629Sraf do {
3995629Sraf new = old = *lockword;
4005629Sraf if (new & SPINNERMASK)
4015629Sraf new -= (1 << SPINNERSHIFT);
4025629Sraf } while (atomic_cas_32(lockword, old, new) != old);
4035629Sraf
4045629Sraf return (new);
4055629Sraf }
4065629Sraf
4075629Sraf /*
4080Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock().
4090Sstevel@tonic-gate * No lock statistics are gathered for these locks.
4105629Sraf * No DTrace probes are provided for these locks.
4110Sstevel@tonic-gate */
4120Sstevel@tonic-gate void
spin_lock_set(mutex_t * mp)4130Sstevel@tonic-gate spin_lock_set(mutex_t *mp)
4140Sstevel@tonic-gate {
4150Sstevel@tonic-gate ulwp_t *self = curthread;
4160Sstevel@tonic-gate
4170Sstevel@tonic-gate no_preempt(self);
4180Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) {
4190Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
4200Sstevel@tonic-gate return;
4210Sstevel@tonic-gate }
4220Sstevel@tonic-gate /*
4230Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock.
4240Sstevel@tonic-gate */
4256247Sraf INCR32(self->ul_spin_lock_spin);
4260Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 ||
4270Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) {
4280Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
4290Sstevel@tonic-gate return;
4300Sstevel@tonic-gate }
4310Sstevel@tonic-gate /*
4320Sstevel@tonic-gate * Try harder if we were previously at a no premption level.
4330Sstevel@tonic-gate */
4340Sstevel@tonic-gate if (self->ul_preempt > 1) {
4356247Sraf INCR32(self->ul_spin_lock_spin2);
4360Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 ||
4370Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) {
4380Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
4390Sstevel@tonic-gate return;
4400Sstevel@tonic-gate }
4410Sstevel@tonic-gate }
4420Sstevel@tonic-gate /*
4430Sstevel@tonic-gate * Give up and block in the kernel for the mutex.
4440Sstevel@tonic-gate */
4456247Sraf INCR32(self->ul_spin_lock_sleep);
446*10887SRoger.Faulkner@Sun.COM (void) ___lwp_mutex_timedlock(mp, NULL, self);
4470Sstevel@tonic-gate }
4480Sstevel@tonic-gate
4490Sstevel@tonic-gate void
spin_lock_clear(mutex_t * mp)4500Sstevel@tonic-gate spin_lock_clear(mutex_t *mp)
4510Sstevel@tonic-gate {
4520Sstevel@tonic-gate ulwp_t *self = curthread;
4530Sstevel@tonic-gate
4540Sstevel@tonic-gate mp->mutex_owner = 0;
4554570Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) {
4564574Sraf (void) ___lwp_mutex_wakeup(mp, 0);
4576247Sraf INCR32(self->ul_spin_lock_wakeup);
4580Sstevel@tonic-gate }
4590Sstevel@tonic-gate preempt(self);
4600Sstevel@tonic-gate }
4610Sstevel@tonic-gate
4620Sstevel@tonic-gate /*
4630Sstevel@tonic-gate * Allocate the sleep queue hash table.
4640Sstevel@tonic-gate */
4650Sstevel@tonic-gate void
queue_alloc(void)4660Sstevel@tonic-gate queue_alloc(void)
4670Sstevel@tonic-gate {
4680Sstevel@tonic-gate ulwp_t *self = curthread;
4690Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
4706247Sraf queue_head_t *qp;
4710Sstevel@tonic-gate void *data;
4720Sstevel@tonic-gate int i;
4730Sstevel@tonic-gate
4740Sstevel@tonic-gate /*
4750Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded.
4760Sstevel@tonic-gate */
4770Sstevel@tonic-gate ASSERT(self == udp->ulwp_one);
4780Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt);
4796515Sraf if ((data = mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t),
4800Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0))
4810Sstevel@tonic-gate == MAP_FAILED)
4820Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table");
4836247Sraf udp->queue_head = qp = (queue_head_t *)data;
4846247Sraf for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
4856247Sraf qp->qh_type = (i < QHASHSIZE)? MX : CV;
4866247Sraf qp->qh_lock.mutex_flag = LOCK_INITED;
4876247Sraf qp->qh_lock.mutex_magic = MUTEX_MAGIC;
4886247Sraf qp->qh_hlist = &qp->qh_def_root;
4896247Sraf #if defined(THREAD_DEBUG)
4906247Sraf qp->qh_hlen = 1;
4916247Sraf qp->qh_hmax = 1;
4926247Sraf #endif
4934574Sraf }
4940Sstevel@tonic-gate }
4950Sstevel@tonic-gate
4960Sstevel@tonic-gate #if defined(THREAD_DEBUG)
4970Sstevel@tonic-gate
4980Sstevel@tonic-gate /*
4990Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue.
5000Sstevel@tonic-gate */
5010Sstevel@tonic-gate void
QVERIFY(queue_head_t * qp)5020Sstevel@tonic-gate QVERIFY(queue_head_t *qp)
5030Sstevel@tonic-gate {
5040Sstevel@tonic-gate ulwp_t *self = curthread;
5050Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
5066247Sraf queue_root_t *qrp;
5070Sstevel@tonic-gate ulwp_t *ulwp;
5080Sstevel@tonic-gate ulwp_t *prev;
5090Sstevel@tonic-gate uint_t index;
5106247Sraf uint32_t cnt;
5110Sstevel@tonic-gate char qtype;
5120Sstevel@tonic-gate void *wchan;
5130Sstevel@tonic-gate
5140Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE);
5150Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
5166247Sraf for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) {
5176247Sraf cnt++;
5186247Sraf ASSERT((qrp->qr_head != NULL && qrp->qr_tail != NULL) ||
5196247Sraf (qrp->qr_head == NULL && qrp->qr_tail == NULL));
5206247Sraf }
5216247Sraf ASSERT(qp->qh_hlen == cnt && qp->qh_hmax >= cnt);
5226247Sraf qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV;
5236247Sraf ASSERT(qp->qh_type == qtype);
5240Sstevel@tonic-gate if (!thread_queue_verify)
5250Sstevel@tonic-gate return;
5260Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */
5276247Sraf for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) {
5286247Sraf for (prev = NULL, ulwp = qrp->qr_head; ulwp != NULL;
5296247Sraf prev = ulwp, ulwp = ulwp->ul_link) {
5306247Sraf cnt++;
5316247Sraf if (ulwp->ul_writer)
5326247Sraf ASSERT(prev == NULL || prev->ul_writer);
5336247Sraf ASSERT(ulwp->ul_qtype == qtype);
5346247Sraf ASSERT(ulwp->ul_wchan != NULL);
5356247Sraf ASSERT(ulwp->ul_sleepq == qp);
5366247Sraf wchan = ulwp->ul_wchan;
5376247Sraf ASSERT(qrp->qr_wchan == wchan);
5386247Sraf index = QUEUE_HASH(wchan, qtype);
5396247Sraf ASSERT(&udp->queue_head[index] == qp);
5406247Sraf }
5416247Sraf ASSERT(qrp->qr_tail == prev);
5420Sstevel@tonic-gate }
5430Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt);
5440Sstevel@tonic-gate }
5450Sstevel@tonic-gate
5460Sstevel@tonic-gate #else /* THREAD_DEBUG */
5470Sstevel@tonic-gate
5480Sstevel@tonic-gate #define QVERIFY(qp)
5490Sstevel@tonic-gate
5500Sstevel@tonic-gate #endif /* THREAD_DEBUG */
5510Sstevel@tonic-gate
5520Sstevel@tonic-gate /*
5530Sstevel@tonic-gate * Acquire a queue head.
5540Sstevel@tonic-gate */
5550Sstevel@tonic-gate queue_head_t *
queue_lock(void * wchan,int qtype)5560Sstevel@tonic-gate queue_lock(void *wchan, int qtype)
5570Sstevel@tonic-gate {
5580Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
5590Sstevel@tonic-gate queue_head_t *qp;
5606247Sraf queue_root_t *qrp;
5610Sstevel@tonic-gate
5620Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV);
5630Sstevel@tonic-gate
5640Sstevel@tonic-gate /*
5650Sstevel@tonic-gate * It is possible that we could be called while still single-threaded.
5660Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array.
5670Sstevel@tonic-gate */
5680Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) {
5690Sstevel@tonic-gate queue_alloc();
5700Sstevel@tonic-gate qp = udp->queue_head;
5710Sstevel@tonic-gate }
5720Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype);
5730Sstevel@tonic-gate spin_lock_set(&qp->qh_lock);
5746247Sraf for (qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next)
5756247Sraf if (qrp->qr_wchan == wchan)
5766247Sraf break;
5776247Sraf if (qrp == NULL && qp->qh_def_root.qr_head == NULL) {
5786247Sraf /* the default queue root is available; use it */
5796247Sraf qrp = &qp->qh_def_root;
5806247Sraf qrp->qr_wchan = wchan;
5816247Sraf ASSERT(qrp->qr_next == NULL);
5826247Sraf ASSERT(qrp->qr_tail == NULL &&
5836247Sraf qrp->qr_rtcount == 0 && qrp->qr_qlen == 0);
5846247Sraf }
5856247Sraf qp->qh_wchan = wchan; /* valid until queue_unlock() is called */
5866247Sraf qp->qh_root = qrp; /* valid until queue_unlock() is called */
5876247Sraf INCR32(qp->qh_lockcount);
5880Sstevel@tonic-gate QVERIFY(qp);
5890Sstevel@tonic-gate return (qp);
5900Sstevel@tonic-gate }
5910Sstevel@tonic-gate
5920Sstevel@tonic-gate /*
5930Sstevel@tonic-gate * Release a queue head.
5940Sstevel@tonic-gate */
5950Sstevel@tonic-gate void
queue_unlock(queue_head_t * qp)5960Sstevel@tonic-gate queue_unlock(queue_head_t *qp)
5970Sstevel@tonic-gate {
5980Sstevel@tonic-gate QVERIFY(qp);
5990Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock);
6000Sstevel@tonic-gate }
6010Sstevel@tonic-gate
6020Sstevel@tonic-gate /*
6030Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the
6040Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half
6050Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below.
6060Sstevel@tonic-gate */
6070Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer)
6080Sstevel@tonic-gate
6090Sstevel@tonic-gate void
enqueue(queue_head_t * qp,ulwp_t * ulwp,int force_fifo)6106247Sraf enqueue(queue_head_t *qp, ulwp_t *ulwp, int force_fifo)
6110Sstevel@tonic-gate {
6126247Sraf queue_root_t *qrp;
6130Sstevel@tonic-gate ulwp_t **ulwpp;
6140Sstevel@tonic-gate ulwp_t *next;
6150Sstevel@tonic-gate int pri = CMP_PRIO(ulwp);
6166247Sraf
6170Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
6180Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp);
6190Sstevel@tonic-gate
6206247Sraf if ((qrp = qp->qh_root) == NULL) {
6216247Sraf /* use the thread's queue root for the linkage */
6226247Sraf qrp = &ulwp->ul_queue_root;
6236247Sraf qrp->qr_next = qp->qh_hlist;
6246247Sraf qrp->qr_prev = NULL;
6256247Sraf qrp->qr_head = NULL;
6266247Sraf qrp->qr_tail = NULL;
6276247Sraf qrp->qr_wchan = qp->qh_wchan;
6286247Sraf qrp->qr_rtcount = 0;
6296247Sraf qrp->qr_qlen = 0;
6306247Sraf qrp->qr_qmax = 0;
6316247Sraf qp->qh_hlist->qr_prev = qrp;
6326247Sraf qp->qh_hlist = qrp;
6336247Sraf qp->qh_root = qrp;
6346247Sraf MAXINCR(qp->qh_hmax, qp->qh_hlen);
6356247Sraf }
6366247Sraf
6370Sstevel@tonic-gate /*
6380Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation,
6390Sstevel@tonic-gate * but it gives better performance for heavily contended locks.
6400Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine
6410Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing:
6420Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO)
6430Sstevel@tonic-gate * 1 : every 128th time
6440Sstevel@tonic-gate * 2 : every 64th time
6450Sstevel@tonic-gate * 3 : every 32nd time
6460Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO)
6470Sstevel@tonic-gate * 5 : every 8th time
6480Sstevel@tonic-gate * 6 : every 4th time
6490Sstevel@tonic-gate * 7 : every 2nd time
6500Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO)
6510Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering.
6520Sstevel@tonic-gate * This breaks live lock conditions that occur in applications
6530Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire
6540Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order.
6556247Sraf * In any event, the queue is maintained in kernel priority order.
6560Sstevel@tonic-gate *
6576247Sraf * If force_fifo is non-zero, fifo queueing is forced.
6580Sstevel@tonic-gate * SUSV3 requires this for semaphores.
6590Sstevel@tonic-gate */
6606247Sraf if (qrp->qr_head == NULL) {
6610Sstevel@tonic-gate /*
6620Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter.
6630Sstevel@tonic-gate */
6646247Sraf ASSERT(qrp->qr_tail == NULL);
6656247Sraf ulwpp = &qrp->qr_head;
6666247Sraf } else if (force_fifo |
6676247Sraf (((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0)) {
6680Sstevel@tonic-gate /*
6690Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater
6700Sstevel@tonic-gate * than or equal to the priority of the thread being queued.
6710Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue.
6720Sstevel@tonic-gate */
6736247Sraf if (pri <= CMP_PRIO(qrp->qr_tail))
6746247Sraf ulwpp = &qrp->qr_tail->ul_link;
6750Sstevel@tonic-gate else {
6766247Sraf for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL;
6770Sstevel@tonic-gate ulwpp = &next->ul_link)
6780Sstevel@tonic-gate if (pri > CMP_PRIO(next))
6790Sstevel@tonic-gate break;
6800Sstevel@tonic-gate }
6810Sstevel@tonic-gate } else {
6820Sstevel@tonic-gate /*
6830Sstevel@tonic-gate * Enqueue before the first thread whose priority is less
6840Sstevel@tonic-gate * than or equal to the priority of the thread being queued.
6850Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue.
6860Sstevel@tonic-gate */
6876247Sraf for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL;
6880Sstevel@tonic-gate ulwpp = &next->ul_link)
6890Sstevel@tonic-gate if (pri >= CMP_PRIO(next))
6900Sstevel@tonic-gate break;
6910Sstevel@tonic-gate }
6920Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL)
6936247Sraf qrp->qr_tail = ulwp;
6940Sstevel@tonic-gate *ulwpp = ulwp;
6950Sstevel@tonic-gate
6960Sstevel@tonic-gate ulwp->ul_sleepq = qp;
6976247Sraf ulwp->ul_wchan = qp->qh_wchan;
6986247Sraf ulwp->ul_qtype = qp->qh_type;
6996247Sraf if ((ulwp->ul_schedctl != NULL &&
7006247Sraf ulwp->ul_schedctl->sc_cid == ulwp->ul_rtclassid) |
7016247Sraf ulwp->ul_pilocks) {
7026247Sraf ulwp->ul_rtqueued = 1;
7036247Sraf qrp->qr_rtcount++;
7046247Sraf }
7056247Sraf MAXINCR(qrp->qr_qmax, qrp->qr_qlen);
7066247Sraf MAXINCR(qp->qh_qmax, qp->qh_qlen);
7076247Sraf }
7086247Sraf
7096247Sraf /*
7106247Sraf * Helper function for queue_slot() and queue_slot_rt().
7116247Sraf * Try to find a non-suspended thread on the queue.
7126247Sraf */
7136247Sraf static ulwp_t **
queue_slot_runnable(ulwp_t ** ulwpp,ulwp_t ** prevp,int rt)7146247Sraf queue_slot_runnable(ulwp_t **ulwpp, ulwp_t **prevp, int rt)
7156247Sraf {
7166247Sraf ulwp_t *ulwp;
7176247Sraf ulwp_t **foundpp = NULL;
7186247Sraf int priority = -1;
7196247Sraf ulwp_t *prev;
7206247Sraf int tpri;
7216247Sraf
7226247Sraf for (prev = NULL;
7236247Sraf (ulwp = *ulwpp) != NULL;
7246247Sraf prev = ulwp, ulwpp = &ulwp->ul_link) {
7256247Sraf if (ulwp->ul_stop) /* skip suspended threads */
7266247Sraf continue;
7276247Sraf tpri = rt? CMP_PRIO(ulwp) : 0;
7286247Sraf if (tpri > priority) {
7296247Sraf foundpp = ulwpp;
7306247Sraf *prevp = prev;
7316247Sraf priority = tpri;
7326247Sraf if (!rt)
7336247Sraf break;
7346247Sraf }
7356247Sraf }
7366247Sraf return (foundpp);
7370Sstevel@tonic-gate }
7380Sstevel@tonic-gate
7390Sstevel@tonic-gate /*
7406247Sraf * For real-time, we search the entire queue because the dispatch
7416247Sraf * (kernel) priorities may have changed since enqueueing.
7420Sstevel@tonic-gate */
7430Sstevel@tonic-gate static ulwp_t **
queue_slot_rt(ulwp_t ** ulwpp_org,ulwp_t ** prevp)7446247Sraf queue_slot_rt(ulwp_t **ulwpp_org, ulwp_t **prevp)
7456247Sraf {
7466247Sraf ulwp_t **ulwpp = ulwpp_org;
7476247Sraf ulwp_t *ulwp = *ulwpp;
7486247Sraf ulwp_t **foundpp = ulwpp;
7496247Sraf int priority = CMP_PRIO(ulwp);
7506247Sraf ulwp_t *prev;
7516247Sraf int tpri;
7526247Sraf
7536247Sraf for (prev = ulwp, ulwpp = &ulwp->ul_link;
7546247Sraf (ulwp = *ulwpp) != NULL;
7556247Sraf prev = ulwp, ulwpp = &ulwp->ul_link) {
7566247Sraf tpri = CMP_PRIO(ulwp);
7576247Sraf if (tpri > priority) {
7586247Sraf foundpp = ulwpp;
7596247Sraf *prevp = prev;
7606247Sraf priority = tpri;
7616247Sraf }
7626247Sraf }
7636247Sraf ulwp = *foundpp;
7646247Sraf
7656247Sraf /*
7666247Sraf * Try not to return a suspended thread.
7676247Sraf * This mimics the old libthread's behavior.
7686247Sraf */
7696247Sraf if (ulwp->ul_stop &&
7706247Sraf (ulwpp = queue_slot_runnable(ulwpp_org, prevp, 1)) != NULL) {
7716247Sraf foundpp = ulwpp;
7726247Sraf ulwp = *foundpp;
7736247Sraf }
7746247Sraf ulwp->ul_rt = 1;
7756247Sraf return (foundpp);
7766247Sraf }
7776247Sraf
7786247Sraf ulwp_t **
queue_slot(queue_head_t * qp,ulwp_t ** prevp,int * more)7796247Sraf queue_slot(queue_head_t *qp, ulwp_t **prevp, int *more)
7806247Sraf {
7816247Sraf queue_root_t *qrp;
7826247Sraf ulwp_t **ulwpp;
7836247Sraf ulwp_t *ulwp;
7846247Sraf int rt;
7856247Sraf
7866247Sraf ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
7876247Sraf
7886247Sraf if ((qrp = qp->qh_root) == NULL || (ulwp = qrp->qr_head) == NULL) {
7896247Sraf *more = 0;
7906247Sraf return (NULL); /* no lwps on the queue */
7916247Sraf }
7926247Sraf rt = (qrp->qr_rtcount != 0);
7936247Sraf *prevp = NULL;
7946247Sraf if (ulwp->ul_link == NULL) { /* only one lwp on the queue */
7956247Sraf *more = 0;
7966247Sraf ulwp->ul_rt = rt;
7976247Sraf return (&qrp->qr_head);
7986247Sraf }
7996247Sraf *more = 1;
8006247Sraf
8016247Sraf if (rt) /* real-time queue */
8026247Sraf return (queue_slot_rt(&qrp->qr_head, prevp));
8036247Sraf /*
8046247Sraf * Try not to return a suspended thread.
8056247Sraf * This mimics the old libthread's behavior.
8066247Sraf */
8076247Sraf if (ulwp->ul_stop &&
8086247Sraf (ulwpp = queue_slot_runnable(&qrp->qr_head, prevp, 0)) != NULL) {
8096247Sraf ulwp = *ulwpp;
8106247Sraf ulwp->ul_rt = 0;
8116247Sraf return (ulwpp);
8126247Sraf }
8136247Sraf /*
8146247Sraf * The common case; just pick the first thread on the queue.
8156247Sraf */
8166247Sraf ulwp->ul_rt = 0;
8176247Sraf return (&qrp->qr_head);
8186247Sraf }
8196247Sraf
8206247Sraf /*
8216247Sraf * Common code for unlinking an lwp from a user-level sleep queue.
8226247Sraf */
8236247Sraf void
queue_unlink(queue_head_t * qp,ulwp_t ** ulwpp,ulwp_t * prev)8246247Sraf queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev)
8256247Sraf {
8266247Sraf queue_root_t *qrp = qp->qh_root;
8276247Sraf queue_root_t *nqrp;
8286247Sraf ulwp_t *ulwp = *ulwpp;
8296247Sraf ulwp_t *next;
8306247Sraf
8316247Sraf ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
8326247Sraf ASSERT(qp->qh_wchan != NULL && ulwp->ul_wchan == qp->qh_wchan);
8336247Sraf
8346247Sraf DECR(qp->qh_qlen);
8356247Sraf DECR(qrp->qr_qlen);
8366247Sraf if (ulwp->ul_rtqueued) {
8376247Sraf ulwp->ul_rtqueued = 0;
8386247Sraf qrp->qr_rtcount--;
8396247Sraf }
8406247Sraf next = ulwp->ul_link;
8416247Sraf *ulwpp = next;
8426247Sraf ulwp->ul_link = NULL;
8436247Sraf if (qrp->qr_tail == ulwp)
8446247Sraf qrp->qr_tail = prev;
8456247Sraf if (qrp == &ulwp->ul_queue_root) {
8466247Sraf /*
8476247Sraf * We can't continue to use the unlinked thread's
8486247Sraf * queue root for the linkage.
8496247Sraf */
8506247Sraf queue_root_t *qr_next = qrp->qr_next;
8516247Sraf queue_root_t *qr_prev = qrp->qr_prev;
8526247Sraf
8536247Sraf if (qrp->qr_tail) {
8546247Sraf /* switch to using the last thread's queue root */
8556247Sraf ASSERT(qrp->qr_qlen != 0);
8566247Sraf nqrp = &qrp->qr_tail->ul_queue_root;
8576247Sraf *nqrp = *qrp;
8586247Sraf if (qr_next)
8596247Sraf qr_next->qr_prev = nqrp;
8606247Sraf if (qr_prev)
8616247Sraf qr_prev->qr_next = nqrp;
8626247Sraf else
8636247Sraf qp->qh_hlist = nqrp;
8646247Sraf qp->qh_root = nqrp;
8656247Sraf } else {
8666247Sraf /* empty queue root; just delete from the hash list */
8676247Sraf ASSERT(qrp->qr_qlen == 0);
8686247Sraf if (qr_next)
8696247Sraf qr_next->qr_prev = qr_prev;
8706247Sraf if (qr_prev)
8716247Sraf qr_prev->qr_next = qr_next;
8726247Sraf else
8736247Sraf qp->qh_hlist = qr_next;
8746247Sraf qp->qh_root = NULL;
8756247Sraf DECR(qp->qh_hlen);
8766247Sraf }
8776247Sraf }
8786247Sraf }
8796247Sraf
8806247Sraf ulwp_t *
dequeue(queue_head_t * qp,int * more)8816247Sraf dequeue(queue_head_t *qp, int *more)
8820Sstevel@tonic-gate {
8830Sstevel@tonic-gate ulwp_t **ulwpp;
8840Sstevel@tonic-gate ulwp_t *ulwp;
8856247Sraf ulwp_t *prev;
8866247Sraf
8876247Sraf if ((ulwpp = queue_slot(qp, &prev, more)) == NULL)
8880Sstevel@tonic-gate return (NULL);
8890Sstevel@tonic-gate ulwp = *ulwpp;
8906247Sraf queue_unlink(qp, ulwpp, prev);
8910Sstevel@tonic-gate ulwp->ul_sleepq = NULL;
8920Sstevel@tonic-gate ulwp->ul_wchan = NULL;
8930Sstevel@tonic-gate return (ulwp);
8940Sstevel@tonic-gate }
8950Sstevel@tonic-gate
8960Sstevel@tonic-gate /*
8970Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan.
8980Sstevel@tonic-gate */
8990Sstevel@tonic-gate ulwp_t *
queue_waiter(queue_head_t * qp)9006247Sraf queue_waiter(queue_head_t *qp)
9010Sstevel@tonic-gate {
9020Sstevel@tonic-gate ulwp_t **ulwpp;
9036247Sraf ulwp_t *prev;
9046247Sraf int more;
9056247Sraf
9066247Sraf if ((ulwpp = queue_slot(qp, &prev, &more)) == NULL)
9070Sstevel@tonic-gate return (NULL);
9080Sstevel@tonic-gate return (*ulwpp);
9090Sstevel@tonic-gate }
9100Sstevel@tonic-gate
9116247Sraf int
dequeue_self(queue_head_t * qp)9126247Sraf dequeue_self(queue_head_t *qp)
9130Sstevel@tonic-gate {
9140Sstevel@tonic-gate ulwp_t *self = curthread;
9156247Sraf queue_root_t *qrp;
9160Sstevel@tonic-gate ulwp_t **ulwpp;
9170Sstevel@tonic-gate ulwp_t *ulwp;
9186247Sraf ulwp_t *prev;
9190Sstevel@tonic-gate int found = 0;
9200Sstevel@tonic-gate
9210Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
9220Sstevel@tonic-gate
9230Sstevel@tonic-gate /* find self on the sleep queue */
9246247Sraf if ((qrp = qp->qh_root) != NULL) {
9256247Sraf for (prev = NULL, ulwpp = &qrp->qr_head;
9266247Sraf (ulwp = *ulwpp) != NULL;
9276247Sraf prev = ulwp, ulwpp = &ulwp->ul_link) {
9286247Sraf if (ulwp == self) {
9296247Sraf queue_unlink(qp, ulwpp, prev);
9306247Sraf self->ul_cvmutex = NULL;
9316247Sraf self->ul_sleepq = NULL;
9326247Sraf self->ul_wchan = NULL;
9336247Sraf found = 1;
9346247Sraf break;
9356247Sraf }
9360Sstevel@tonic-gate }
9370Sstevel@tonic-gate }
9380Sstevel@tonic-gate
9390Sstevel@tonic-gate if (!found)
9400Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue");
9410Sstevel@tonic-gate
9426247Sraf return ((qrp = qp->qh_root) != NULL && qrp->qr_head != NULL);
9430Sstevel@tonic-gate }
9440Sstevel@tonic-gate
9450Sstevel@tonic-gate /*
9460Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take
9470Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks.
9480Sstevel@tonic-gate */
9490Sstevel@tonic-gate void
unsleep_self(void)9500Sstevel@tonic-gate unsleep_self(void)
9510Sstevel@tonic-gate {
9520Sstevel@tonic-gate ulwp_t *self = curthread;
9530Sstevel@tonic-gate queue_head_t *qp;
9540Sstevel@tonic-gate
9550Sstevel@tonic-gate /*
9560Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead
9570Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly.
9580Sstevel@tonic-gate */
9590Sstevel@tonic-gate self->ul_critical++;
9600Sstevel@tonic-gate while (self->ul_sleepq != NULL) {
9610Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype);
9620Sstevel@tonic-gate /*
9630Sstevel@tonic-gate * We may have been moved from a CV queue to a
9640Sstevel@tonic-gate * mutex queue while we were attempting queue_lock().
9650Sstevel@tonic-gate * If so, just loop around and try again.
9660Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq.
9670Sstevel@tonic-gate */
9686247Sraf if (qp == self->ul_sleepq)
9696247Sraf (void) dequeue_self(qp);
9700Sstevel@tonic-gate queue_unlock(qp);
9710Sstevel@tonic-gate }
9726247Sraf self->ul_writer = 0;
9730Sstevel@tonic-gate self->ul_critical--;
9740Sstevel@tonic-gate }
9750Sstevel@tonic-gate
9760Sstevel@tonic-gate /*
9770Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call.
9780Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly.
9790Sstevel@tonic-gate */
9804574Sraf static int
mutex_lock_kernel(mutex_t * mp,timespec_t * tsp,tdb_mutex_stats_t * msp)9810Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
9820Sstevel@tonic-gate {
9830Sstevel@tonic-gate ulwp_t *self = curthread;
9840Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
9854574Sraf int mtype = mp->mutex_type;
9860Sstevel@tonic-gate hrtime_t begin_sleep;
9874574Sraf int acquired;
9880Sstevel@tonic-gate int error;
9890Sstevel@tonic-gate
9900Sstevel@tonic-gate self->ul_sp = stkptr();
9910Sstevel@tonic-gate self->ul_wchan = mp;
9920Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) {
9930Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP;
9940Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp;
9950Sstevel@tonic-gate tdb_event(TD_SLEEP, udp);
9960Sstevel@tonic-gate }
9970Sstevel@tonic-gate if (msp) {
9980Sstevel@tonic-gate tdb_incr(msp->mutex_sleep);
9990Sstevel@tonic-gate begin_sleep = gethrtime();
10000Sstevel@tonic-gate }
10010Sstevel@tonic-gate
10020Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp);
10030Sstevel@tonic-gate
10040Sstevel@tonic-gate for (;;) {
10054574Sraf /*
10064574Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED
10074574Sraf * means we successfully acquired the lock.
10084574Sraf */
1009*10887SRoger.Faulkner@Sun.COM if ((error = ___lwp_mutex_timedlock(mp, tsp, self)) != 0 &&
10104574Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) {
10114574Sraf acquired = 0;
10120Sstevel@tonic-gate break;
10130Sstevel@tonic-gate }
10140Sstevel@tonic-gate
10154574Sraf if (mtype & USYNC_PROCESS) {
10160Sstevel@tonic-gate /*
10170Sstevel@tonic-gate * Defend against forkall(). We may be the child,
10180Sstevel@tonic-gate * in which case we don't actually own the mutex.
10190Sstevel@tonic-gate */
10200Sstevel@tonic-gate enter_critical(self);
10210Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) {
10220Sstevel@tonic-gate exit_critical(self);
10234574Sraf acquired = 1;
10240Sstevel@tonic-gate break;
10250Sstevel@tonic-gate }
10260Sstevel@tonic-gate exit_critical(self);
10270Sstevel@tonic-gate } else {
10284574Sraf acquired = 1;
10290Sstevel@tonic-gate break;
10300Sstevel@tonic-gate }
10310Sstevel@tonic-gate }
10327907SRoger.Faulkner@Sun.COM
10330Sstevel@tonic-gate if (msp)
10340Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep;
10350Sstevel@tonic-gate self->ul_wchan = NULL;
10360Sstevel@tonic-gate self->ul_sp = 0;
10370Sstevel@tonic-gate
10384574Sraf if (acquired) {
1039*10887SRoger.Faulkner@Sun.COM ASSERT(mp->mutex_owner == (uintptr_t)self);
10404574Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
10414574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
10424574Sraf } else {
10434574Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
10444574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error);
10454574Sraf }
10464574Sraf
10470Sstevel@tonic-gate return (error);
10480Sstevel@tonic-gate }
10490Sstevel@tonic-gate
10500Sstevel@tonic-gate /*
10510Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call.
10520Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly.
10530Sstevel@tonic-gate */
10540Sstevel@tonic-gate int
mutex_trylock_kernel(mutex_t * mp)10550Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp)
10560Sstevel@tonic-gate {
10570Sstevel@tonic-gate ulwp_t *self = curthread;
10580Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
10594574Sraf int mtype = mp->mutex_type;
10600Sstevel@tonic-gate int error;
10614574Sraf int acquired;
10620Sstevel@tonic-gate
10630Sstevel@tonic-gate for (;;) {
10644574Sraf /*
10654574Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED
10664574Sraf * means we successfully acquired the lock.
10674574Sraf */
1068*10887SRoger.Faulkner@Sun.COM if ((error = ___lwp_mutex_trylock(mp, self)) != 0 &&
10694574Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) {
10704574Sraf acquired = 0;
10710Sstevel@tonic-gate break;
10720Sstevel@tonic-gate }
10730Sstevel@tonic-gate
10744574Sraf if (mtype & USYNC_PROCESS) {
10750Sstevel@tonic-gate /*
10760Sstevel@tonic-gate * Defend against forkall(). We may be the child,
10770Sstevel@tonic-gate * in which case we don't actually own the mutex.
10780Sstevel@tonic-gate */
10790Sstevel@tonic-gate enter_critical(self);
10800Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) {
10810Sstevel@tonic-gate exit_critical(self);
10824574Sraf acquired = 1;
10830Sstevel@tonic-gate break;
10840Sstevel@tonic-gate }
10850Sstevel@tonic-gate exit_critical(self);
10860Sstevel@tonic-gate } else {
10874574Sraf acquired = 1;
10880Sstevel@tonic-gate break;
10890Sstevel@tonic-gate }
10900Sstevel@tonic-gate }
10910Sstevel@tonic-gate
10924574Sraf if (acquired) {
1093*10887SRoger.Faulkner@Sun.COM ASSERT(mp->mutex_owner == (uintptr_t)self);
10944574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
10954574Sraf } else if (error != EBUSY) {
10964574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error);
10974574Sraf }
10984574Sraf
10990Sstevel@tonic-gate return (error);
11000Sstevel@tonic-gate }
11010Sstevel@tonic-gate
11020Sstevel@tonic-gate volatile sc_shared_t *
setup_schedctl(void)11030Sstevel@tonic-gate setup_schedctl(void)
11040Sstevel@tonic-gate {
11050Sstevel@tonic-gate ulwp_t *self = curthread;
11060Sstevel@tonic-gate volatile sc_shared_t *scp;
11070Sstevel@tonic-gate sc_shared_t *tmp;
11080Sstevel@tonic-gate
11090Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */
11100Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */
11110Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */
11120Sstevel@tonic-gate enter_critical(self);
11130Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags;
11140Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1))
11150Sstevel@tonic-gate self->ul_schedctl = scp = tmp;
11160Sstevel@tonic-gate exit_critical(self);
11170Sstevel@tonic-gate }
11180Sstevel@tonic-gate /*
11190Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded
11200Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address
11210Sstevel@tonic-gate * we are returning could be invalid due to a forkall()
11220Sstevel@tonic-gate * having occurred in another thread.
11230Sstevel@tonic-gate */
11240Sstevel@tonic-gate return (scp);
11250Sstevel@tonic-gate }
11260Sstevel@tonic-gate
11270Sstevel@tonic-gate /*
11280Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc.
11290Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc.
11300Sstevel@tonic-gate */
11316812Sraf #pragma weak schedctl_lookup = schedctl_init
11320Sstevel@tonic-gate schedctl_t *
schedctl_init(void)11336812Sraf schedctl_init(void)
11340Sstevel@tonic-gate {
11350Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl();
11360Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl);
11370Sstevel@tonic-gate }
11380Sstevel@tonic-gate
11390Sstevel@tonic-gate void
schedctl_exit(void)11406812Sraf schedctl_exit(void)
11410Sstevel@tonic-gate {
11420Sstevel@tonic-gate }
11430Sstevel@tonic-gate
11440Sstevel@tonic-gate /*
11450Sstevel@tonic-gate * Contract private interface for java.
11460Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet.
11470Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data.
11480Sstevel@tonic-gate */
11490Sstevel@tonic-gate volatile sc_shared_t *volatile *
_thr_schedctl(void)11500Sstevel@tonic-gate _thr_schedctl(void)
11510Sstevel@tonic-gate {
11520Sstevel@tonic-gate ulwp_t *self = curthread;
11530Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr;
11540Sstevel@tonic-gate
11550Sstevel@tonic-gate if (self->ul_vfork)
11560Sstevel@tonic-gate return (NULL);
11570Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL)
11580Sstevel@tonic-gate (void) setup_schedctl();
11590Sstevel@tonic-gate return (ptr);
11600Sstevel@tonic-gate }
11610Sstevel@tonic-gate
11620Sstevel@tonic-gate /*
11630Sstevel@tonic-gate * Block signals and attempt to block preemption.
11640Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested.
11650Sstevel@tonic-gate */
11660Sstevel@tonic-gate void
no_preempt(ulwp_t * self)11670Sstevel@tonic-gate no_preempt(ulwp_t *self)
11680Sstevel@tonic-gate {
11690Sstevel@tonic-gate volatile sc_shared_t *scp;
11700Sstevel@tonic-gate
11710Sstevel@tonic-gate if (self->ul_preempt++ == 0) {
11720Sstevel@tonic-gate enter_critical(self);
11730Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL ||
11740Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) {
11750Sstevel@tonic-gate /*
11760Sstevel@tonic-gate * Save the pre-existing preempt value.
11770Sstevel@tonic-gate */
11780Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt;
11790Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1;
11800Sstevel@tonic-gate }
11810Sstevel@tonic-gate }
11820Sstevel@tonic-gate }
11830Sstevel@tonic-gate
11840Sstevel@tonic-gate /*
11850Sstevel@tonic-gate * Undo the effects of no_preempt().
11860Sstevel@tonic-gate */
11870Sstevel@tonic-gate void
preempt(ulwp_t * self)11880Sstevel@tonic-gate preempt(ulwp_t *self)
11890Sstevel@tonic-gate {
11900Sstevel@tonic-gate volatile sc_shared_t *scp;
11910Sstevel@tonic-gate
11920Sstevel@tonic-gate ASSERT(self->ul_preempt > 0);
11930Sstevel@tonic-gate if (--self->ul_preempt == 0) {
11940Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) {
11950Sstevel@tonic-gate /*
11960Sstevel@tonic-gate * Restore the pre-existing preempt value.
11970Sstevel@tonic-gate */
11980Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt;
11990Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield &&
12000Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) {
12016515Sraf yield();
12020Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) {
12030Sstevel@tonic-gate /*
12040Sstevel@tonic-gate * Shouldn't happen. This is either
12050Sstevel@tonic-gate * a race condition or the thread
12060Sstevel@tonic-gate * just entered the real-time class.
12070Sstevel@tonic-gate */
12086515Sraf yield();
12090Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0;
12100Sstevel@tonic-gate }
12110Sstevel@tonic-gate }
12120Sstevel@tonic-gate }
12130Sstevel@tonic-gate exit_critical(self);
12140Sstevel@tonic-gate }
12150Sstevel@tonic-gate }
12160Sstevel@tonic-gate
12170Sstevel@tonic-gate /*
12180Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to
12190Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified
12200Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the
12210Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat
12220Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical
12230Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below.
12240Sstevel@tonic-gate */
12250Sstevel@tonic-gate static lwpid_t
preempt_unpark(ulwp_t * self,lwpid_t lwpid)12260Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid)
12270Sstevel@tonic-gate {
12280Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl;
12290Sstevel@tonic-gate
12300Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0);
12310Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) ||
12320Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) {
12330Sstevel@tonic-gate (void) __lwp_unpark(lwpid);
12340Sstevel@tonic-gate lwpid = 0;
12350Sstevel@tonic-gate }
12360Sstevel@tonic-gate return (lwpid);
12370Sstevel@tonic-gate }
12380Sstevel@tonic-gate
12390Sstevel@tonic-gate /*
12404613Sraf * Spin for a while (if 'tryhard' is true), trying to grab the lock.
12410Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it.
12420Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread.
12430Sstevel@tonic-gate */
12444574Sraf static int
mutex_trylock_adaptive(mutex_t * mp,int tryhard)12454613Sraf mutex_trylock_adaptive(mutex_t *mp, int tryhard)
12460Sstevel@tonic-gate {
12470Sstevel@tonic-gate ulwp_t *self = curthread;
12484574Sraf int error = EBUSY;
12490Sstevel@tonic-gate ulwp_t *ulwp;
12500Sstevel@tonic-gate volatile sc_shared_t *scp;
12515629Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
12525629Sraf volatile uint64_t *ownerp = (volatile uint64_t *)&mp->mutex_owner;
12535629Sraf uint32_t new_lockword;
12545629Sraf int count = 0;
12555629Sraf int max_count;
12565629Sraf uint8_t max_spinners;
12574574Sraf
12584574Sraf ASSERT(!(mp->mutex_type & USYNC_PROCESS));
12594574Sraf
12607907SRoger.Faulkner@Sun.COM if (MUTEX_OWNED(mp, self))
12610Sstevel@tonic-gate return (EBUSY);
12620Sstevel@tonic-gate
12637907SRoger.Faulkner@Sun.COM enter_critical(self);
12647907SRoger.Faulkner@Sun.COM
12654574Sraf /* short-cut, not definitive (see below) */
12664574Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
12674574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
12685629Sraf error = ENOTRECOVERABLE;
12695629Sraf goto done;
12704574Sraf }
12714574Sraf
12725629Sraf /*
12735629Sraf * Make one attempt to acquire the lock before
12745629Sraf * incurring the overhead of the spin loop.
12755629Sraf */
12765629Sraf if (set_lock_byte(lockp) == 0) {
12775629Sraf *ownerp = (uintptr_t)self;
12785629Sraf error = 0;
12795629Sraf goto done;
12805629Sraf }
12815629Sraf if (!tryhard)
12825629Sraf goto done;
12835629Sraf if (ncpus == 0)
12845629Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
12855629Sraf if ((max_spinners = self->ul_max_spinners) >= ncpus)
12865629Sraf max_spinners = ncpus - 1;
12875629Sraf max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0;
12885629Sraf if (max_count == 0)
12895629Sraf goto done;
12905629Sraf
12910Sstevel@tonic-gate /*
12920Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into
12930Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex.
12940Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended
12950Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken
12960Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention.
12970Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written
12980Sstevel@tonic-gate * programs will not suffer in any case.
12990Sstevel@tonic-gate */
13007907SRoger.Faulkner@Sun.COM if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1)
13015629Sraf goto done;
13025629Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp);
13035629Sraf for (count = 1; ; count++) {
13040Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) {
13050Sstevel@tonic-gate *ownerp = (uintptr_t)self;
13064574Sraf error = 0;
13074574Sraf break;
13080Sstevel@tonic-gate }
13095629Sraf if (count == max_count)
13105629Sraf break;
13110Sstevel@tonic-gate SMT_PAUSE();
13120Sstevel@tonic-gate /*
13130Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on
13140Sstevel@tonic-gate * a processor; it will not drop the lock any time soon
13150Sstevel@tonic-gate * and we would just be wasting time to keep spinning.
13160Sstevel@tonic-gate *
13170Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t)
13180Sstevel@tonic-gate * without ensuring that the other thread does not exit.
13190Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being
13200Sstevel@tonic-gate * deallocated by the library (the library employs a free
13210Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads
13220Sstevel@tonic-gate * are created) and on schedctl shared memory never being
13230Sstevel@tonic-gate * deallocated once created via __schedctl().
13240Sstevel@tonic-gate *
13250Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread
13260Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking
13270Sstevel@tonic-gate * at some other thread's schedctl data. This almost never
13280Sstevel@tonic-gate * happens and is benign when it does.
13290Sstevel@tonic-gate */
13300Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
13310Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL ||
13320Sstevel@tonic-gate scp->sc_state != SC_ONPROC))
13330Sstevel@tonic-gate break;
13340Sstevel@tonic-gate }
13355629Sraf new_lockword = spinners_decr(&mp->mutex_lockword);
13365629Sraf if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) {
13375629Sraf /*
13385629Sraf * We haven't yet acquired the lock, the lock
13395629Sraf * is free, and there are no other spinners.
13405629Sraf * Make one final attempt to acquire the lock.
13415629Sraf *
13425629Sraf * This isn't strictly necessary since mutex_lock_queue()
13435629Sraf * (the next action this thread will take if it doesn't
13445629Sraf * acquire the lock here) makes one attempt to acquire
13455629Sraf * the lock before putting the thread to sleep.
13465629Sraf *
13475629Sraf * If the next action for this thread (on failure here)
13485629Sraf * were not to call mutex_lock_queue(), this would be
13495629Sraf * necessary for correctness, to avoid ending up with an
13505629Sraf * unheld mutex with waiters but no one to wake them up.
13515629Sraf */
13525629Sraf if (set_lock_byte(lockp) == 0) {
13535629Sraf *ownerp = (uintptr_t)self;
13545629Sraf error = 0;
13555629Sraf }
13565629Sraf count++;
13575629Sraf }
13580Sstevel@tonic-gate
13595629Sraf done:
13604574Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
13614574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
13624574Sraf /*
13636057Sraf * We shouldn't own the mutex.
13646057Sraf * Just clear the lock; everyone has already been waked up.
13654574Sraf */
13667907SRoger.Faulkner@Sun.COM *ownerp = 0;
13676057Sraf (void) clear_lockbyte(&mp->mutex_lockword);
13684574Sraf error = ENOTRECOVERABLE;
13694574Sraf }
13704574Sraf
13717907SRoger.Faulkner@Sun.COM exit_critical(self);
13727907SRoger.Faulkner@Sun.COM
13734574Sraf if (error) {
13745629Sraf if (count) {
13759397SJonathan.Haslam@Sun.COM DTRACE_PROBE3(plockstat, mutex__spun, mp, 0, count);
13765629Sraf }
13774574Sraf if (error != EBUSY) {
13784574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error);
13794574Sraf }
13804574Sraf } else {
13815629Sraf if (count) {
13829397SJonathan.Haslam@Sun.COM DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
13835629Sraf }
13844574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
13854574Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) {
13864574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
13874574Sraf error = EOWNERDEAD;
13884574Sraf }
13894574Sraf }
13904574Sraf
13914574Sraf return (error);
13920Sstevel@tonic-gate }
13930Sstevel@tonic-gate
13940Sstevel@tonic-gate /*
13950Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks.
13960Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it.
13970Sstevel@tonic-gate */
13984574Sraf static int
mutex_queuelock_adaptive(mutex_t * mp)13990Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp)
14000Sstevel@tonic-gate {
14010Sstevel@tonic-gate ulwp_t *ulwp;
14020Sstevel@tonic-gate volatile sc_shared_t *scp;
14030Sstevel@tonic-gate volatile uint8_t *lockp;
14040Sstevel@tonic-gate volatile uint64_t *ownerp;
14050Sstevel@tonic-gate int count = curthread->ul_queue_spin;
14060Sstevel@tonic-gate
14070Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD);
14080Sstevel@tonic-gate
14090Sstevel@tonic-gate if (count == 0)
14100Sstevel@tonic-gate return (EBUSY);
14110Sstevel@tonic-gate
14120Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw;
14130Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner;
14140Sstevel@tonic-gate while (--count >= 0) {
14150Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0)
14160Sstevel@tonic-gate return (0);
14170Sstevel@tonic-gate SMT_PAUSE();
14180Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
14190Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL ||
14200Sstevel@tonic-gate scp->sc_state != SC_ONPROC))
14210Sstevel@tonic-gate break;
14220Sstevel@tonic-gate }
14230Sstevel@tonic-gate
14240Sstevel@tonic-gate return (EBUSY);
14250Sstevel@tonic-gate }
14260Sstevel@tonic-gate
14270Sstevel@tonic-gate /*
14280Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes.
14294613Sraf * Spin for a while (if 'tryhard' is true), trying to grab the lock.
14300Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it.
14310Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread
14320Sstevel@tonic-gate * and mutex_ownerpid set to the current pid.
14330Sstevel@tonic-gate */
14344574Sraf static int
mutex_trylock_process(mutex_t * mp,int tryhard)14354613Sraf mutex_trylock_process(mutex_t *mp, int tryhard)
14360Sstevel@tonic-gate {
14370Sstevel@tonic-gate ulwp_t *self = curthread;
14385629Sraf uberdata_t *udp = self->ul_uberdata;
14394574Sraf int error = EBUSY;
14406057Sraf volatile uint64_t *lockp = (volatile uint64_t *)&mp->mutex_lockword64;
14415629Sraf uint32_t new_lockword;
14425629Sraf int count = 0;
14435629Sraf int max_count;
14445629Sraf uint8_t max_spinners;
14454574Sraf
14467255Sraf #if defined(__sparc) && !defined(_LP64)
14477255Sraf /* horrible hack, necessary only on 32-bit sparc */
14487255Sraf int fix_alignment_problem =
14497255Sraf (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
14507255Sraf self->ul_misaligned && !(mp->mutex_type & LOCK_ROBUST));
14517255Sraf #endif
14527255Sraf
14534574Sraf ASSERT(mp->mutex_type & USYNC_PROCESS);
14544574Sraf
14554574Sraf if (shared_mutex_held(mp))
14560Sstevel@tonic-gate return (EBUSY);
14570Sstevel@tonic-gate
14587907SRoger.Faulkner@Sun.COM enter_critical(self);
14597907SRoger.Faulkner@Sun.COM
14604574Sraf /* short-cut, not definitive (see below) */
14614574Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
14624574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
14635629Sraf error = ENOTRECOVERABLE;
14645629Sraf goto done;
14654574Sraf }
14664574Sraf
14675629Sraf /*
14685629Sraf * Make one attempt to acquire the lock before
14695629Sraf * incurring the overhead of the spin loop.
14705629Sraf */
14717255Sraf #if defined(__sparc) && !defined(_LP64)
14727255Sraf /* horrible hack, necessary only on 32-bit sparc */
14737255Sraf if (fix_alignment_problem) {
14747255Sraf if (set_lock_byte(&mp->mutex_lockw) == 0) {
14757255Sraf mp->mutex_ownerpid = udp->pid;
14767255Sraf mp->mutex_owner = (uintptr_t)self;
14777255Sraf error = 0;
14787255Sraf goto done;
14797255Sraf }
14807255Sraf } else
14817255Sraf #endif
14826057Sraf if (set_lock_byte64(lockp, udp->pid) == 0) {
14835629Sraf mp->mutex_owner = (uintptr_t)self;
14846057Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */
14855629Sraf error = 0;
14865629Sraf goto done;
14875629Sraf }
14885629Sraf if (!tryhard)
14895629Sraf goto done;
14904574Sraf if (ncpus == 0)
14914574Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
14925629Sraf if ((max_spinners = self->ul_max_spinners) >= ncpus)
14935629Sraf max_spinners = ncpus - 1;
14945629Sraf max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0;
14955629Sraf if (max_count == 0)
14965629Sraf goto done;
14975629Sraf
14980Sstevel@tonic-gate /*
14990Sstevel@tonic-gate * This is a process-shared mutex.
15000Sstevel@tonic-gate * We cannot know if the owner is running on a processor.
15010Sstevel@tonic-gate * We just spin and hope that it is on a processor.
15020Sstevel@tonic-gate */
15037907SRoger.Faulkner@Sun.COM if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1)
15045629Sraf goto done;
15055629Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp);
15065629Sraf for (count = 1; ; count++) {
15077255Sraf #if defined(__sparc) && !defined(_LP64)
15087255Sraf /* horrible hack, necessary only on 32-bit sparc */
15097255Sraf if (fix_alignment_problem) {
15107255Sraf if ((*lockp & LOCKMASK64) == 0 &&
15117255Sraf set_lock_byte(&mp->mutex_lockw) == 0) {
15127255Sraf mp->mutex_ownerpid = udp->pid;
15137255Sraf mp->mutex_owner = (uintptr_t)self;
15147255Sraf error = 0;
15157255Sraf break;
15167255Sraf }
15177255Sraf } else
15187255Sraf #endif
15196057Sraf if ((*lockp & LOCKMASK64) == 0 &&
15206057Sraf set_lock_byte64(lockp, udp->pid) == 0) {
15214574Sraf mp->mutex_owner = (uintptr_t)self;
15226057Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */
15234574Sraf error = 0;
15244574Sraf break;
15254574Sraf }
15265629Sraf if (count == max_count)
15275629Sraf break;
15284574Sraf SMT_PAUSE();
15294574Sraf }
15305629Sraf new_lockword = spinners_decr(&mp->mutex_lockword);
15315629Sraf if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) {
15325629Sraf /*
15335629Sraf * We haven't yet acquired the lock, the lock
15345629Sraf * is free, and there are no other spinners.
15355629Sraf * Make one final attempt to acquire the lock.
15365629Sraf *
15375629Sraf * This isn't strictly necessary since mutex_lock_kernel()
15385629Sraf * (the next action this thread will take if it doesn't
15395629Sraf * acquire the lock here) makes one attempt to acquire
15405629Sraf * the lock before putting the thread to sleep.
15415629Sraf *
15425629Sraf * If the next action for this thread (on failure here)
15435629Sraf * were not to call mutex_lock_kernel(), this would be
15445629Sraf * necessary for correctness, to avoid ending up with an
15455629Sraf * unheld mutex with waiters but no one to wake them up.
15465629Sraf */
15477255Sraf #if defined(__sparc) && !defined(_LP64)
15487255Sraf /* horrible hack, necessary only on 32-bit sparc */
15497255Sraf if (fix_alignment_problem) {
15507255Sraf if (set_lock_byte(&mp->mutex_lockw) == 0) {
15517255Sraf mp->mutex_ownerpid = udp->pid;
15527255Sraf mp->mutex_owner = (uintptr_t)self;
15537255Sraf error = 0;
15547255Sraf }
15557255Sraf } else
15567255Sraf #endif
15576057Sraf if (set_lock_byte64(lockp, udp->pid) == 0) {
15585629Sraf mp->mutex_owner = (uintptr_t)self;
15596057Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */
15605629Sraf error = 0;
15615629Sraf }
15625629Sraf count++;
15635629Sraf }
15644574Sraf
15655629Sraf done:
15664574Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
15674574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
15684574Sraf /*
15696057Sraf * We shouldn't own the mutex.
15706057Sraf * Just clear the lock; everyone has already been waked up.
15714574Sraf */
15724574Sraf mp->mutex_owner = 0;
15736057Sraf /* mp->mutex_ownerpid is cleared by clear_lockbyte64() */
15746057Sraf (void) clear_lockbyte64(&mp->mutex_lockword64);
15754574Sraf error = ENOTRECOVERABLE;
15760Sstevel@tonic-gate }
15770Sstevel@tonic-gate
15787907SRoger.Faulkner@Sun.COM exit_critical(self);
15797907SRoger.Faulkner@Sun.COM
15804574Sraf if (error) {
15815629Sraf if (count) {
15829397SJonathan.Haslam@Sun.COM DTRACE_PROBE3(plockstat, mutex__spun, mp, 0, count);
15835629Sraf }
15844574Sraf if (error != EBUSY) {
15854574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error);
15864574Sraf }
15874574Sraf } else {
15885629Sraf if (count) {
15899397SJonathan.Haslam@Sun.COM DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
15905629Sraf }
15914574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
15924574Sraf if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
15934574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
15944574Sraf if (mp->mutex_flag & LOCK_OWNERDEAD)
15954574Sraf error = EOWNERDEAD;
15964574Sraf else if (mp->mutex_type & USYNC_PROCESS_ROBUST)
15974574Sraf error = ELOCKUNMAPPED;
15984574Sraf else
15994574Sraf error = EOWNERDEAD;
16004574Sraf }
16014574Sraf }
16024574Sraf
16034574Sraf return (error);
16040Sstevel@tonic-gate }
16050Sstevel@tonic-gate
16060Sstevel@tonic-gate /*
16070Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex.
16080Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any.
16090Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
16100Sstevel@tonic-gate * to wake up the specified lwp.
16110Sstevel@tonic-gate */
16124574Sraf static lwpid_t
mutex_wakeup(mutex_t * mp)16130Sstevel@tonic-gate mutex_wakeup(mutex_t *mp)
16140Sstevel@tonic-gate {
16150Sstevel@tonic-gate lwpid_t lwpid = 0;
16166247Sraf int more;
16170Sstevel@tonic-gate queue_head_t *qp;
16180Sstevel@tonic-gate ulwp_t *ulwp;
16190Sstevel@tonic-gate
16200Sstevel@tonic-gate /*
16210Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex
16220Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex
16230Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose.
16240Sstevel@tonic-gate */
16250Sstevel@tonic-gate qp = queue_lock(mp, MX);
16266247Sraf if ((ulwp = dequeue(qp, &more)) != NULL) {
16270Sstevel@tonic-gate lwpid = ulwp->ul_lwpid;
16286247Sraf mp->mutex_waiters = more;
16290Sstevel@tonic-gate }
16300Sstevel@tonic-gate queue_unlock(qp);
16310Sstevel@tonic-gate return (lwpid);
16320Sstevel@tonic-gate }
16330Sstevel@tonic-gate
16340Sstevel@tonic-gate /*
16354574Sraf * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex.
16364574Sraf */
16374574Sraf static void
mutex_wakeup_all(mutex_t * mp)16384574Sraf mutex_wakeup_all(mutex_t *mp)
16394574Sraf {
16404574Sraf queue_head_t *qp;
16416247Sraf queue_root_t *qrp;
16424574Sraf int nlwpid = 0;
16434574Sraf int maxlwps = MAXLWPS;
16444574Sraf ulwp_t *ulwp;
16454574Sraf lwpid_t buffer[MAXLWPS];
16464574Sraf lwpid_t *lwpid = buffer;
16474574Sraf
16484574Sraf /*
16494574Sraf * Walk the list of waiters and prepare to wake up all of them.
16504574Sraf * The waiters flag has already been cleared from the mutex.
16514574Sraf *
16524574Sraf * We keep track of lwpids that are to be unparked in lwpid[].
16534574Sraf * __lwp_unpark_all() is called to unpark all of them after
16544574Sraf * they have been removed from the sleep queue and the sleep
16554574Sraf * queue lock has been dropped. If we run out of space in our
16564574Sraf * on-stack buffer, we need to allocate more but we can't call
16574574Sraf * lmalloc() because we are holding a queue lock when the overflow
16584574Sraf * occurs and lmalloc() acquires a lock. We can't use alloca()
16594574Sraf * either because the application may have allocated a small
16604574Sraf * stack and we don't want to overrun the stack. So we call
16614574Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap()
16624574Sraf * system call directly since that path acquires no locks.
16634574Sraf */
16644574Sraf qp = queue_lock(mp, MX);
16656247Sraf for (;;) {
16666247Sraf if ((qrp = qp->qh_root) == NULL ||
16676247Sraf (ulwp = qrp->qr_head) == NULL)
16686247Sraf break;
16696247Sraf ASSERT(ulwp->ul_wchan == mp);
16706247Sraf queue_unlink(qp, &qrp->qr_head, NULL);
16716247Sraf ulwp->ul_sleepq = NULL;
16726247Sraf ulwp->ul_wchan = NULL;
16736247Sraf if (nlwpid == maxlwps)
16746247Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
16756247Sraf lwpid[nlwpid++] = ulwp->ul_lwpid;
16764574Sraf }
16774574Sraf
16784574Sraf if (nlwpid == 0) {
16794574Sraf queue_unlock(qp);
16804574Sraf } else {
16815629Sraf mp->mutex_waiters = 0;
16824574Sraf no_preempt(curthread);
16834574Sraf queue_unlock(qp);
16844574Sraf if (nlwpid == 1)
16854574Sraf (void) __lwp_unpark(lwpid[0]);
16864574Sraf else
16874574Sraf (void) __lwp_unpark_all(lwpid, nlwpid);
16884574Sraf preempt(curthread);
16894574Sraf }
16904574Sraf
16914574Sraf if (lwpid != buffer)
16926515Sraf (void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
16934574Sraf }
16944574Sraf
16954574Sraf /*
16965629Sraf * Release a process-private mutex.
16975629Sraf * As an optimization, if there are waiters but there are also spinners
16985629Sraf * attempting to acquire the mutex, then don't bother waking up a waiter;
16995629Sraf * one of the spinners will acquire the mutex soon and it would be a waste
17005629Sraf * of resources to wake up some thread just to have it spin for a while
17015629Sraf * and then possibly go back to sleep. See mutex_trylock_adaptive().
17020Sstevel@tonic-gate */
17034574Sraf static lwpid_t
mutex_unlock_queue(mutex_t * mp,int release_all)17044574Sraf mutex_unlock_queue(mutex_t *mp, int release_all)
17050Sstevel@tonic-gate {
17067907SRoger.Faulkner@Sun.COM ulwp_t *self = curthread;
17075629Sraf lwpid_t lwpid = 0;
17085629Sraf uint32_t old_lockword;
17095629Sraf
17106057Sraf DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
17117907SRoger.Faulkner@Sun.COM sigoff(self);
17125629Sraf mp->mutex_owner = 0;
17135629Sraf old_lockword = clear_lockbyte(&mp->mutex_lockword);
17145629Sraf if ((old_lockword & WAITERMASK) &&
17155629Sraf (release_all || (old_lockword & SPINNERMASK) == 0)) {
17160Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */
17175629Sraf if (release_all)
17185629Sraf mutex_wakeup_all(mp);
17195629Sraf else
17205629Sraf lwpid = mutex_wakeup(mp);
17215629Sraf if (lwpid == 0)
17225629Sraf preempt(self);
17234574Sraf }
17247907SRoger.Faulkner@Sun.COM sigon(self);
17250Sstevel@tonic-gate return (lwpid);
17260Sstevel@tonic-gate }
17270Sstevel@tonic-gate
17280Sstevel@tonic-gate /*
17290Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes.
17300Sstevel@tonic-gate */
17314574Sraf static void
mutex_unlock_process(mutex_t * mp,int release_all)17324574Sraf mutex_unlock_process(mutex_t *mp, int release_all)
17330Sstevel@tonic-gate {
17347255Sraf ulwp_t *self = curthread;
17356057Sraf uint64_t old_lockword64;
17366057Sraf
17376057Sraf DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
17387907SRoger.Faulkner@Sun.COM sigoff(self);
17390Sstevel@tonic-gate mp->mutex_owner = 0;
17407255Sraf #if defined(__sparc) && !defined(_LP64)
17417255Sraf /* horrible hack, necessary only on 32-bit sparc */
17427255Sraf if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
17437255Sraf self->ul_misaligned && !(mp->mutex_type & LOCK_ROBUST)) {
17447255Sraf uint32_t old_lockword;
17457255Sraf mp->mutex_ownerpid = 0;
17467255Sraf old_lockword = clear_lockbyte(&mp->mutex_lockword);
17477255Sraf if ((old_lockword & WAITERMASK) &&
17487255Sraf (release_all || (old_lockword & SPINNERMASK) == 0)) {
17497255Sraf no_preempt(self); /* ensure a prompt wakeup */
17507255Sraf (void) ___lwp_mutex_wakeup(mp, release_all);
17517255Sraf preempt(self);
17527255Sraf }
17537907SRoger.Faulkner@Sun.COM sigon(self);
17547255Sraf return;
17557255Sraf }
17567255Sraf #endif
17576057Sraf /* mp->mutex_ownerpid is cleared by clear_lockbyte64() */
17586057Sraf old_lockword64 = clear_lockbyte64(&mp->mutex_lockword64);
17596057Sraf if ((old_lockword64 & WAITERMASK64) &&
17606057Sraf (release_all || (old_lockword64 & SPINNERMASK64) == 0)) {
17615629Sraf no_preempt(self); /* ensure a prompt wakeup */
17625629Sraf (void) ___lwp_mutex_wakeup(mp, release_all);
17635629Sraf preempt(self);
17640Sstevel@tonic-gate }
17657907SRoger.Faulkner@Sun.COM sigon(self);
17660Sstevel@tonic-gate }
17670Sstevel@tonic-gate
17680Sstevel@tonic-gate void
stall(void)17690Sstevel@tonic-gate stall(void)
17700Sstevel@tonic-gate {
17710Sstevel@tonic-gate for (;;)
17720Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL);
17730Sstevel@tonic-gate }
17740Sstevel@tonic-gate
17750Sstevel@tonic-gate /*
17760Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues.
17770Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
17784574Sraf * If successful, returns with mutex_owner set correctly.
17790Sstevel@tonic-gate */
17800Sstevel@tonic-gate int
mutex_lock_queue(ulwp_t * self,tdb_mutex_stats_t * msp,mutex_t * mp,timespec_t * tsp)17810Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
17820Sstevel@tonic-gate timespec_t *tsp)
17830Sstevel@tonic-gate {
17840Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
17850Sstevel@tonic-gate queue_head_t *qp;
17860Sstevel@tonic-gate hrtime_t begin_sleep;
17870Sstevel@tonic-gate int error = 0;
17880Sstevel@tonic-gate
17890Sstevel@tonic-gate self->ul_sp = stkptr();
17900Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) {
17910Sstevel@tonic-gate self->ul_wchan = mp;
17920Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP;
17930Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp;
17940Sstevel@tonic-gate tdb_event(TD_SLEEP, udp);
17950Sstevel@tonic-gate }
17960Sstevel@tonic-gate if (msp) {
17970Sstevel@tonic-gate tdb_incr(msp->mutex_sleep);
17980Sstevel@tonic-gate begin_sleep = gethrtime();
17990Sstevel@tonic-gate }
18000Sstevel@tonic-gate
18010Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp);
18020Sstevel@tonic-gate
18030Sstevel@tonic-gate /*
18040Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are
18050Sstevel@tonic-gate * unable to grab the lock, go park in the kernel.
18060Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock.
18070Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock.
18080Sstevel@tonic-gate */
18090Sstevel@tonic-gate qp = queue_lock(mp, MX);
18106247Sraf enqueue(qp, self, 0);
18110Sstevel@tonic-gate mp->mutex_waiters = 1;
18120Sstevel@tonic-gate for (;;) {
18130Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) {
18140Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
18156247Sraf mp->mutex_waiters = dequeue_self(qp);
18160Sstevel@tonic-gate break;
18170Sstevel@tonic-gate }
18180Sstevel@tonic-gate set_parking_flag(self, 1);
18190Sstevel@tonic-gate queue_unlock(qp);
18200Sstevel@tonic-gate /*
18210Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp
18220Sstevel@tonic-gate * if we are unparked before the timeout expires.
18230Sstevel@tonic-gate */
18245629Sraf error = __lwp_park(tsp, 0);
18250Sstevel@tonic-gate set_parking_flag(self, 0);
18260Sstevel@tonic-gate /*
18270Sstevel@tonic-gate * We could have taken a signal or suspended ourself.
18280Sstevel@tonic-gate * If we did, then we removed ourself from the queue.
18290Sstevel@tonic-gate * Someone else may have removed us from the queue
18300Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have
18310Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still
18320Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup.
18330Sstevel@tonic-gate */
18340Sstevel@tonic-gate qp = queue_lock(mp, MX);
18350Sstevel@tonic-gate if (self->ul_sleepq == NULL) {
18365629Sraf if (error) {
18376247Sraf mp->mutex_waiters = queue_waiter(qp)? 1 : 0;
18385629Sraf if (error != EINTR)
18395629Sraf break;
18405629Sraf error = 0;
18415629Sraf }
18420Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) {
18430Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
18440Sstevel@tonic-gate break;
18450Sstevel@tonic-gate }
18466247Sraf enqueue(qp, self, 0);
18470Sstevel@tonic-gate mp->mutex_waiters = 1;
18480Sstevel@tonic-gate }
18490Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp &&
18500Sstevel@tonic-gate self->ul_qtype == MX &&
18510Sstevel@tonic-gate self->ul_wchan == mp);
18520Sstevel@tonic-gate if (error) {
18535629Sraf if (error != EINTR) {
18546247Sraf mp->mutex_waiters = dequeue_self(qp);
18555629Sraf break;
18565629Sraf }
18575629Sraf error = 0;
18580Sstevel@tonic-gate }
18590Sstevel@tonic-gate }
18600Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
18610Sstevel@tonic-gate self->ul_wchan == NULL);
18620Sstevel@tonic-gate self->ul_sp = 0;
18630Sstevel@tonic-gate
18640Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME);
18654574Sraf
18664574Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
18674574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
18684574Sraf /*
18696057Sraf * We shouldn't own the mutex.
18706057Sraf * Just clear the lock; everyone has already been waked up.
18714574Sraf */
18724574Sraf mp->mutex_owner = 0;
18736057Sraf (void) clear_lockbyte(&mp->mutex_lockword);
18744574Sraf error = ENOTRECOVERABLE;
18754574Sraf }
18764574Sraf
18777907SRoger.Faulkner@Sun.COM queue_unlock(qp);
18787907SRoger.Faulkner@Sun.COM
18797907SRoger.Faulkner@Sun.COM if (msp)
18807907SRoger.Faulkner@Sun.COM msp->mutex_sleep_time += gethrtime() - begin_sleep;
18817907SRoger.Faulkner@Sun.COM
18824574Sraf if (error) {
18834574Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
18844574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error);
18854574Sraf } else {
18864574Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
18874574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
18884574Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) {
18894574Sraf ASSERT(mp->mutex_type & LOCK_ROBUST);
18904574Sraf error = EOWNERDEAD;
18914574Sraf }
18924574Sraf }
18934574Sraf
18940Sstevel@tonic-gate return (error);
18950Sstevel@tonic-gate }
18960Sstevel@tonic-gate
18974574Sraf static int
mutex_recursion(mutex_t * mp,int mtype,int try)18984574Sraf mutex_recursion(mutex_t *mp, int mtype, int try)
18994574Sraf {
19006812Sraf ASSERT(mutex_held(mp));
19014574Sraf ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK));
19024574Sraf ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
19034574Sraf
19044574Sraf if (mtype & LOCK_RECURSIVE) {
19054574Sraf if (mp->mutex_rcount == RECURSION_MAX) {
19064574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN);
19074574Sraf return (EAGAIN);
19084574Sraf }
19094574Sraf mp->mutex_rcount++;
19104574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
19114574Sraf return (0);
19124574Sraf }
19134574Sraf if (try == MUTEX_LOCK) {
19144574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
19154574Sraf return (EDEADLK);
19164574Sraf }
19174574Sraf return (EBUSY);
19184574Sraf }
19194574Sraf
19204574Sraf /*
19214574Sraf * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so
19224574Sraf * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary.
19234574Sraf * We use tdb_hash_lock here and in the synch object tracking code in
19244574Sraf * the tdb_agent.c file. There is no conflict between these two usages.
19254574Sraf */
19264574Sraf void
register_lock(mutex_t * mp)19274574Sraf register_lock(mutex_t *mp)
19284574Sraf {
19294574Sraf uberdata_t *udp = curthread->ul_uberdata;
19304574Sraf uint_t hash = LOCK_HASH(mp);
19314574Sraf robust_t *rlp;
19329170SRoger.Faulkner@Sun.COM robust_t *invalid;
19334574Sraf robust_t **rlpp;
19344574Sraf robust_t **table;
19354574Sraf
19364574Sraf if ((table = udp->robustlocks) == NULL) {
19374574Sraf lmutex_lock(&udp->tdb_hash_lock);
19384574Sraf if ((table = udp->robustlocks) == NULL) {
19394574Sraf table = lmalloc(LOCKHASHSZ * sizeof (robust_t *));
19406812Sraf membar_producer();
19414574Sraf udp->robustlocks = table;
19424574Sraf }
19434574Sraf lmutex_unlock(&udp->tdb_hash_lock);
19444574Sraf }
19456812Sraf membar_consumer();
19464574Sraf
19474574Sraf /*
19484574Sraf * First search the registered table with no locks held.
19494574Sraf * This is safe because the table never shrinks
19504574Sraf * and we can only get a false negative.
19514574Sraf */
19524574Sraf for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) {
19534574Sraf if (rlp->robust_lock == mp) /* already registered */
19544574Sraf return;
19554574Sraf }
19564574Sraf
19574574Sraf /*
19584574Sraf * The lock was not found.
19594574Sraf * Repeat the operation with tdb_hash_lock held.
19604574Sraf */
19614574Sraf lmutex_lock(&udp->tdb_hash_lock);
19624574Sraf
19639170SRoger.Faulkner@Sun.COM invalid = NULL;
19644574Sraf for (rlpp = &table[hash];
19654574Sraf (rlp = *rlpp) != NULL;
19664574Sraf rlpp = &rlp->robust_next) {
19674574Sraf if (rlp->robust_lock == mp) { /* already registered */
19684574Sraf lmutex_unlock(&udp->tdb_hash_lock);
19694574Sraf return;
19704574Sraf }
19719170SRoger.Faulkner@Sun.COM /* remember the first invalid entry, if any */
19729170SRoger.Faulkner@Sun.COM if (rlp->robust_lock == INVALID_ADDR && invalid == NULL)
19739170SRoger.Faulkner@Sun.COM invalid = rlp;
19744574Sraf }
19754574Sraf
19764574Sraf /*
19774574Sraf * The lock has never been registered.
19789170SRoger.Faulkner@Sun.COM * Add it to the table and register it now.
19794574Sraf */
19809264SRoger.Faulkner@Sun.COM if ((rlp = invalid) != NULL) {
19819170SRoger.Faulkner@Sun.COM /*
19829170SRoger.Faulkner@Sun.COM * Reuse the invalid entry we found above.
19839170SRoger.Faulkner@Sun.COM * The linkages are still correct.
19849170SRoger.Faulkner@Sun.COM */
19859264SRoger.Faulkner@Sun.COM rlp->robust_lock = mp;
19869170SRoger.Faulkner@Sun.COM membar_producer();
19879170SRoger.Faulkner@Sun.COM } else {
19889170SRoger.Faulkner@Sun.COM /*
19899170SRoger.Faulkner@Sun.COM * Allocate a new entry and add it to
19909170SRoger.Faulkner@Sun.COM * the hash table and to the global list.
19919170SRoger.Faulkner@Sun.COM */
19929170SRoger.Faulkner@Sun.COM rlp = lmalloc(sizeof (*rlp));
19939170SRoger.Faulkner@Sun.COM rlp->robust_lock = mp;
19949170SRoger.Faulkner@Sun.COM rlp->robust_next = NULL;
19959170SRoger.Faulkner@Sun.COM rlp->robust_list = udp->robustlist;
19969170SRoger.Faulkner@Sun.COM udp->robustlist = rlp;
19979170SRoger.Faulkner@Sun.COM membar_producer();
19989170SRoger.Faulkner@Sun.COM *rlpp = rlp;
19999170SRoger.Faulkner@Sun.COM }
20009170SRoger.Faulkner@Sun.COM
20019170SRoger.Faulkner@Sun.COM lmutex_unlock(&udp->tdb_hash_lock);
20029170SRoger.Faulkner@Sun.COM
20039264SRoger.Faulkner@Sun.COM (void) ___lwp_mutex_register(mp, &rlp->robust_lock);
20044574Sraf }
20054574Sraf
20064574Sraf /*
20074574Sraf * This is called in the child of fork()/forkall() to start over
20084574Sraf * with a clean slate. (Each process must register its own locks.)
20094574Sraf * No locks are needed because all other threads are suspended or gone.
20104574Sraf */
20114574Sraf void
unregister_locks(void)20129264SRoger.Faulkner@Sun.COM unregister_locks(void)
20134574Sraf {
20144574Sraf uberdata_t *udp = curthread->ul_uberdata;
20154574Sraf robust_t **table;
20164574Sraf robust_t *rlp;
20174574Sraf robust_t *next;
20184574Sraf
20199170SRoger.Faulkner@Sun.COM /*
20209170SRoger.Faulkner@Sun.COM * Do this first, before calling lfree().
20219170SRoger.Faulkner@Sun.COM */
20229170SRoger.Faulkner@Sun.COM table = udp->robustlocks;
20239170SRoger.Faulkner@Sun.COM udp->robustlocks = NULL;
20249170SRoger.Faulkner@Sun.COM rlp = udp->robustlist;
20259170SRoger.Faulkner@Sun.COM udp->robustlist = NULL;
20269170SRoger.Faulkner@Sun.COM
20279170SRoger.Faulkner@Sun.COM /*
20289264SRoger.Faulkner@Sun.COM * Do this by traversing the global list, not the hash table.
20299170SRoger.Faulkner@Sun.COM */
20309170SRoger.Faulkner@Sun.COM while (rlp != NULL) {
20319170SRoger.Faulkner@Sun.COM next = rlp->robust_list;
20329170SRoger.Faulkner@Sun.COM lfree(rlp, sizeof (*rlp));
20339170SRoger.Faulkner@Sun.COM rlp = next;
20349170SRoger.Faulkner@Sun.COM }
20359170SRoger.Faulkner@Sun.COM if (table != NULL)
20364574Sraf lfree(table, LOCKHASHSZ * sizeof (robust_t *));
20374574Sraf }
20384574Sraf
20390Sstevel@tonic-gate /*
20400Sstevel@tonic-gate * Returns with mutex_owner set correctly.
20410Sstevel@tonic-gate */
20426247Sraf int
mutex_lock_internal(mutex_t * mp,timespec_t * tsp,int try)20430Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
20440Sstevel@tonic-gate {
20450Sstevel@tonic-gate ulwp_t *self = curthread;
20460Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
20470Sstevel@tonic-gate int mtype = mp->mutex_type;
20480Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
20490Sstevel@tonic-gate int error = 0;
20506247Sraf int noceil = try & MUTEX_NOCEIL;
20514574Sraf uint8_t ceil;
20524574Sraf int myprio;
20530Sstevel@tonic-gate
20546247Sraf try &= ~MUTEX_NOCEIL;
20550Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
20560Sstevel@tonic-gate
20570Sstevel@tonic-gate if (!self->ul_schedctl_called)
20580Sstevel@tonic-gate (void) setup_schedctl();
20590Sstevel@tonic-gate
20600Sstevel@tonic-gate if (msp && try == MUTEX_TRY)
20610Sstevel@tonic-gate tdb_incr(msp->mutex_try);
20620Sstevel@tonic-gate
20636812Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_held(mp))
20644574Sraf return (mutex_recursion(mp, mtype, try));
20650Sstevel@tonic-gate
20660Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK &&
20676812Sraf tsp == NULL && mutex_held(mp))
20680Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL);
20690Sstevel@tonic-gate
20706247Sraf if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) {
20716247Sraf update_sched(self);
20726247Sraf if (self->ul_cid != self->ul_rtclassid) {
20736247Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EPERM);
20746247Sraf return (EPERM);
20756247Sraf }
20764574Sraf ceil = mp->mutex_ceiling;
20776247Sraf myprio = self->ul_epri? self->ul_epri : self->ul_pri;
20784574Sraf if (myprio > ceil) {
20794574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL);
20804574Sraf return (EINVAL);
20814574Sraf }
20824574Sraf if ((error = _ceil_mylist_add(mp)) != 0) {
20834574Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error);
20844574Sraf return (error);
20850Sstevel@tonic-gate }
20864574Sraf if (myprio < ceil)
20874574Sraf _ceil_prio_inherit(ceil);
20884574Sraf }
20894574Sraf
20904574Sraf if ((mtype & (USYNC_PROCESS | LOCK_ROBUST))
20914574Sraf == (USYNC_PROCESS | LOCK_ROBUST))
20924574Sraf register_lock(mp);
20934574Sraf
20944574Sraf if (mtype & LOCK_PRIO_INHERIT) {
20954574Sraf /* go straight to the kernel */
20964574Sraf if (try == MUTEX_TRY)
20974574Sraf error = mutex_trylock_kernel(mp);
20984574Sraf else /* MUTEX_LOCK */
20994574Sraf error = mutex_lock_kernel(mp, tsp, msp);
21004574Sraf /*
21014574Sraf * The kernel never sets or clears the lock byte
21024574Sraf * for LOCK_PRIO_INHERIT mutexes.
21034574Sraf * Set it here for consistency.
21044574Sraf */
21054574Sraf switch (error) {
21064574Sraf case 0:
21076247Sraf self->ul_pilocks++;
21084574Sraf mp->mutex_lockw = LOCKSET;
21094574Sraf break;
21104574Sraf case EOWNERDEAD:
21114574Sraf case ELOCKUNMAPPED:
21126247Sraf self->ul_pilocks++;
21134574Sraf mp->mutex_lockw = LOCKSET;
21144574Sraf /* FALLTHROUGH */
21154574Sraf case ENOTRECOVERABLE:
21164574Sraf ASSERT(mtype & LOCK_ROBUST);
21174574Sraf break;
21184574Sraf case EDEADLK:
21197376SRoger.Faulkner@Sun.COM if (try == MUTEX_TRY) {
21207376SRoger.Faulkner@Sun.COM error = EBUSY;
21217376SRoger.Faulkner@Sun.COM } else if (tsp != NULL) { /* simulate a timeout */
21227376SRoger.Faulkner@Sun.COM /*
21237376SRoger.Faulkner@Sun.COM * Note: mutex_timedlock() never returns EINTR.
21247376SRoger.Faulkner@Sun.COM */
21257376SRoger.Faulkner@Sun.COM timespec_t ts = *tsp;
21267376SRoger.Faulkner@Sun.COM timespec_t rts;
21277376SRoger.Faulkner@Sun.COM
21287376SRoger.Faulkner@Sun.COM while (__nanosleep(&ts, &rts) == EINTR)
21297376SRoger.Faulkner@Sun.COM ts = rts;
21307376SRoger.Faulkner@Sun.COM error = ETIME;
21317376SRoger.Faulkner@Sun.COM } else { /* simulate a deadlock */
21324574Sraf stall();
21337376SRoger.Faulkner@Sun.COM }
21344574Sraf break;
21350Sstevel@tonic-gate }
21360Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) {
21374613Sraf error = mutex_trylock_process(mp, try == MUTEX_LOCK);
21384574Sraf if (error == EBUSY && try == MUTEX_LOCK)
21390Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp);
21405629Sraf } else { /* USYNC_THREAD */
21414613Sraf error = mutex_trylock_adaptive(mp, try == MUTEX_LOCK);
21424574Sraf if (error == EBUSY && try == MUTEX_LOCK)
21434574Sraf error = mutex_lock_queue(self, msp, mp, tsp);
21440Sstevel@tonic-gate }
21450Sstevel@tonic-gate
21460Sstevel@tonic-gate switch (error) {
21474574Sraf case 0:
21480Sstevel@tonic-gate case EOWNERDEAD:
21490Sstevel@tonic-gate case ELOCKUNMAPPED:
21504574Sraf if (mtype & LOCK_ROBUST)
21514574Sraf remember_lock(mp);
21520Sstevel@tonic-gate if (msp)
21530Sstevel@tonic-gate record_begin_hold(msp);
21540Sstevel@tonic-gate break;
21550Sstevel@tonic-gate default:
21566247Sraf if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) {
21574574Sraf (void) _ceil_mylist_del(mp);
21584574Sraf if (myprio < ceil)
21594574Sraf _ceil_prio_waive();
21604574Sraf }
21610Sstevel@tonic-gate if (try == MUTEX_TRY) {
21620Sstevel@tonic-gate if (msp)
21630Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail);
21640Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) {
21650Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
21660Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp);
21670Sstevel@tonic-gate }
21680Sstevel@tonic-gate }
21690Sstevel@tonic-gate break;
21700Sstevel@tonic-gate }
21710Sstevel@tonic-gate
21720Sstevel@tonic-gate return (error);
21730Sstevel@tonic-gate }
21740Sstevel@tonic-gate
21750Sstevel@tonic-gate int
fast_process_lock(mutex_t * mp,timespec_t * tsp,int mtype,int try)21760Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
21770Sstevel@tonic-gate {
21780Sstevel@tonic-gate ulwp_t *self = curthread;
21790Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
21800Sstevel@tonic-gate
21810Sstevel@tonic-gate /*
21820Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that
21830Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and
21840Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set.
21850Sstevel@tonic-gate */
21864574Sraf ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0);
21870Sstevel@tonic-gate enter_critical(self);
21887255Sraf #if defined(__sparc) && !defined(_LP64)
21897255Sraf /* horrible hack, necessary only on 32-bit sparc */
21907255Sraf if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
21917255Sraf self->ul_misaligned) {
21927255Sraf if (set_lock_byte(&mp->mutex_lockw) == 0) {
21937255Sraf mp->mutex_ownerpid = udp->pid;
21947255Sraf mp->mutex_owner = (uintptr_t)self;
21957255Sraf exit_critical(self);
21967255Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
21977255Sraf return (0);
21987255Sraf }
21997255Sraf } else
22007255Sraf #endif
22016057Sraf if (set_lock_byte64(&mp->mutex_lockword64, udp->pid) == 0) {
22020Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
22036057Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */
22040Sstevel@tonic-gate exit_critical(self);
22050Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
22060Sstevel@tonic-gate return (0);
22070Sstevel@tonic-gate }
22080Sstevel@tonic-gate exit_critical(self);
22090Sstevel@tonic-gate
22104574Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp))
22114574Sraf return (mutex_recursion(mp, mtype, try));
22124574Sraf
22134613Sraf if (try == MUTEX_LOCK) {
22144613Sraf if (mutex_trylock_process(mp, 1) == 0)
22154613Sraf return (0);
22160Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL));
22174613Sraf }
22180Sstevel@tonic-gate
22190Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) {
22200Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
22210Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp);
22220Sstevel@tonic-gate }
22230Sstevel@tonic-gate return (EBUSY);
22240Sstevel@tonic-gate }
22250Sstevel@tonic-gate
22260Sstevel@tonic-gate static int
mutex_lock_impl(mutex_t * mp,timespec_t * tsp)22270Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
22280Sstevel@tonic-gate {
22290Sstevel@tonic-gate ulwp_t *self = curthread;
22306247Sraf int mtype = mp->mutex_type;
22310Sstevel@tonic-gate uberflags_t *gflags;
22320Sstevel@tonic-gate
22337255Sraf if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
22347255Sraf self->ul_error_detection && self->ul_misaligned == 0)
22357255Sraf lock_error(mp, "mutex_lock", NULL, "mutex is misaligned");
22367255Sraf
22370Sstevel@tonic-gate /*
22380Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including
22390Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
22400Sstevel@tonic-gate * no error detection, no lock statistics,
22410Sstevel@tonic-gate * and the process has only a single thread.
22420Sstevel@tonic-gate * (Most likely a traditional single-threaded application.)
22430Sstevel@tonic-gate */
22446247Sraf if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
22456247Sraf self->ul_uberdata->uberflags.uf_all) == 0) {
22460Sstevel@tonic-gate /*
22470Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation.
22487907SRoger.Faulkner@Sun.COM * We do, however, need to protect against signals.
22490Sstevel@tonic-gate */
22500Sstevel@tonic-gate if (mp->mutex_lockw == 0) {
22517907SRoger.Faulkner@Sun.COM sigoff(self);
22520Sstevel@tonic-gate mp->mutex_lockw = LOCKSET;
22530Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
22547907SRoger.Faulkner@Sun.COM sigon(self);
22550Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
22560Sstevel@tonic-gate return (0);
22570Sstevel@tonic-gate }
22584574Sraf if (mtype && MUTEX_OWNER(mp) == self)
22594574Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK));
22600Sstevel@tonic-gate /*
22610Sstevel@tonic-gate * We have reached a deadlock, probably because the
22620Sstevel@tonic-gate * process is executing non-async-signal-safe code in
22630Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock
22640Sstevel@tonic-gate * that it already owns. This is not surprising, given
22650Sstevel@tonic-gate * bad programming practices over the years that has
22660Sstevel@tonic-gate * resulted in applications calling printf() and such
22670Sstevel@tonic-gate * in their signal handlers. Unless the user has told
22680Sstevel@tonic-gate * us that the signal handlers are safe by setting:
22690Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1
22700Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking.
22710Sstevel@tonic-gate */
22720Sstevel@tonic-gate if (tsp == NULL &&
22730Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) {
22740Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
22750Sstevel@tonic-gate return (EDEADLK);
22760Sstevel@tonic-gate }
22770Sstevel@tonic-gate }
22780Sstevel@tonic-gate
22790Sstevel@tonic-gate /*
22800Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
22810Sstevel@tonic-gate * no error detection, and no lock statistics.
22820Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
22830Sstevel@tonic-gate */
22840Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL &&
22850Sstevel@tonic-gate (gflags->uf_trs_ted |
22860Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
22870Sstevel@tonic-gate if (mtype & USYNC_PROCESS)
22880Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
22897907SRoger.Faulkner@Sun.COM sigoff(self);
22900Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) {
22910Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
22927907SRoger.Faulkner@Sun.COM sigon(self);
22930Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
22940Sstevel@tonic-gate return (0);
22950Sstevel@tonic-gate }
22967907SRoger.Faulkner@Sun.COM sigon(self);
22974574Sraf if (mtype && MUTEX_OWNER(mp) == self)
22984574Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK));
22994613Sraf if (mutex_trylock_adaptive(mp, 1) != 0)
23004574Sraf return (mutex_lock_queue(self, NULL, mp, tsp));
23014574Sraf return (0);
23020Sstevel@tonic-gate }
23030Sstevel@tonic-gate
23040Sstevel@tonic-gate /* else do it the long way */
23050Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK));
23060Sstevel@tonic-gate }
23070Sstevel@tonic-gate
23086812Sraf #pragma weak pthread_mutex_lock = mutex_lock
23096812Sraf #pragma weak _mutex_lock = mutex_lock
23100Sstevel@tonic-gate int
mutex_lock(mutex_t * mp)23116812Sraf mutex_lock(mutex_t *mp)
23120Sstevel@tonic-gate {
23130Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23140Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL));
23150Sstevel@tonic-gate }
23160Sstevel@tonic-gate
23170Sstevel@tonic-gate int
pthread_mutex_timedlock(pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD abstime)23186812Sraf pthread_mutex_timedlock(pthread_mutex_t *_RESTRICT_KYWD mp,
23196812Sraf const struct timespec *_RESTRICT_KYWD abstime)
23200Sstevel@tonic-gate {
23210Sstevel@tonic-gate timespec_t tslocal;
23220Sstevel@tonic-gate int error;
23230Sstevel@tonic-gate
23240Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23250Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
23266812Sraf error = mutex_lock_impl((mutex_t *)mp, &tslocal);
23270Sstevel@tonic-gate if (error == ETIME)
23280Sstevel@tonic-gate error = ETIMEDOUT;
23290Sstevel@tonic-gate return (error);
23300Sstevel@tonic-gate }
23310Sstevel@tonic-gate
23320Sstevel@tonic-gate int
pthread_mutex_reltimedlock_np(pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD reltime)23336812Sraf pthread_mutex_reltimedlock_np(pthread_mutex_t *_RESTRICT_KYWD mp,
23346812Sraf const struct timespec *_RESTRICT_KYWD reltime)
23350Sstevel@tonic-gate {
23360Sstevel@tonic-gate timespec_t tslocal;
23370Sstevel@tonic-gate int error;
23380Sstevel@tonic-gate
23390Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23400Sstevel@tonic-gate tslocal = *reltime;
23416812Sraf error = mutex_lock_impl((mutex_t *)mp, &tslocal);
23420Sstevel@tonic-gate if (error == ETIME)
23430Sstevel@tonic-gate error = ETIMEDOUT;
23440Sstevel@tonic-gate return (error);
23450Sstevel@tonic-gate }
23460Sstevel@tonic-gate
23476812Sraf #pragma weak pthread_mutex_trylock = mutex_trylock
23480Sstevel@tonic-gate int
mutex_trylock(mutex_t * mp)23496812Sraf mutex_trylock(mutex_t *mp)
23500Sstevel@tonic-gate {
23510Sstevel@tonic-gate ulwp_t *self = curthread;
23520Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
23536247Sraf int mtype = mp->mutex_type;
23540Sstevel@tonic-gate uberflags_t *gflags;
23550Sstevel@tonic-gate
23560Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23576247Sraf
23580Sstevel@tonic-gate /*
23590Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including
23600Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
23610Sstevel@tonic-gate * no error detection, no lock statistics,
23620Sstevel@tonic-gate * and the process has only a single thread.
23630Sstevel@tonic-gate * (Most likely a traditional single-threaded application.)
23640Sstevel@tonic-gate */
23656247Sraf if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
23660Sstevel@tonic-gate udp->uberflags.uf_all) == 0) {
23670Sstevel@tonic-gate /*
23680Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation.
23697907SRoger.Faulkner@Sun.COM * We do, however, need to protect against signals.
23700Sstevel@tonic-gate */
23710Sstevel@tonic-gate if (mp->mutex_lockw == 0) {
23727907SRoger.Faulkner@Sun.COM sigoff(self);
23730Sstevel@tonic-gate mp->mutex_lockw = LOCKSET;
23740Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
23757907SRoger.Faulkner@Sun.COM sigon(self);
23760Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
23770Sstevel@tonic-gate return (0);
23780Sstevel@tonic-gate }
23794574Sraf if (mtype && MUTEX_OWNER(mp) == self)
23804574Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY));
23810Sstevel@tonic-gate return (EBUSY);
23820Sstevel@tonic-gate }
23830Sstevel@tonic-gate
23840Sstevel@tonic-gate /*
23850Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
23860Sstevel@tonic-gate * no error detection, and no lock statistics.
23870Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
23880Sstevel@tonic-gate */
23890Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL &&
23900Sstevel@tonic-gate (gflags->uf_trs_ted |
23910Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
23920Sstevel@tonic-gate if (mtype & USYNC_PROCESS)
23930Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
23947907SRoger.Faulkner@Sun.COM sigoff(self);
23950Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) {
23960Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
23977907SRoger.Faulkner@Sun.COM sigon(self);
23980Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
23990Sstevel@tonic-gate return (0);
24000Sstevel@tonic-gate }
24017907SRoger.Faulkner@Sun.COM sigon(self);
24024574Sraf if (mtype && MUTEX_OWNER(mp) == self)
24034574Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY));
24044613Sraf if (__td_event_report(self, TD_LOCK_TRY, udp)) {
24054613Sraf self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
24064613Sraf tdb_event(TD_LOCK_TRY, udp);
24070Sstevel@tonic-gate }
24084613Sraf return (EBUSY);
24090Sstevel@tonic-gate }
24100Sstevel@tonic-gate
24110Sstevel@tonic-gate /* else do it the long way */
24120Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY));
24130Sstevel@tonic-gate }
24140Sstevel@tonic-gate
24150Sstevel@tonic-gate int
mutex_unlock_internal(mutex_t * mp,int retain_robust_flags)24164574Sraf mutex_unlock_internal(mutex_t *mp, int retain_robust_flags)
24170Sstevel@tonic-gate {
24180Sstevel@tonic-gate ulwp_t *self = curthread;
24190Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
24200Sstevel@tonic-gate int mtype = mp->mutex_type;
24210Sstevel@tonic-gate tdb_mutex_stats_t *msp;
24224574Sraf int error = 0;
24234574Sraf int release_all;
24240Sstevel@tonic-gate lwpid_t lwpid;
24250Sstevel@tonic-gate
24268036SRoger.Faulkner@Sun.COM if ((mtype & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
24278036SRoger.Faulkner@Sun.COM !mutex_held(mp))
24280Sstevel@tonic-gate return (EPERM);
24290Sstevel@tonic-gate
24306812Sraf if (self->ul_error_detection && !mutex_held(mp))
24310Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL);
24320Sstevel@tonic-gate
24330Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
24340Sstevel@tonic-gate mp->mutex_rcount--;
24350Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
24360Sstevel@tonic-gate return (0);
24370Sstevel@tonic-gate }
24380Sstevel@tonic-gate
24390Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL)
24400Sstevel@tonic-gate (void) record_hold_time(msp);
24410Sstevel@tonic-gate
24424574Sraf if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) &&
24434574Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
24448036SRoger.Faulkner@Sun.COM ASSERT(mtype & LOCK_ROBUST);
24454574Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
24464574Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE;
24474574Sraf }
24484574Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
24494574Sraf
24504574Sraf if (mtype & LOCK_PRIO_INHERIT) {
24510Sstevel@tonic-gate no_preempt(self);
24520Sstevel@tonic-gate mp->mutex_owner = 0;
24536057Sraf /* mp->mutex_ownerpid is cleared by ___lwp_mutex_unlock() */
24540Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
24554574Sraf mp->mutex_lockw = LOCKCLEAR;
24566247Sraf self->ul_pilocks--;
24574574Sraf error = ___lwp_mutex_unlock(mp);
24580Sstevel@tonic-gate preempt(self);
24590Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) {
24605629Sraf mutex_unlock_process(mp, release_all);
24610Sstevel@tonic-gate } else { /* USYNC_THREAD */
24624574Sraf if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) {
24630Sstevel@tonic-gate (void) __lwp_unpark(lwpid);
24640Sstevel@tonic-gate preempt(self);
24650Sstevel@tonic-gate }
24660Sstevel@tonic-gate }
24670Sstevel@tonic-gate
24684574Sraf if (mtype & LOCK_ROBUST)
24694574Sraf forget_lock(mp);
24704574Sraf
24714574Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
24724574Sraf _ceil_prio_waive();
24734574Sraf
24740Sstevel@tonic-gate return (error);
24750Sstevel@tonic-gate }
24760Sstevel@tonic-gate
24776812Sraf #pragma weak pthread_mutex_unlock = mutex_unlock
24786812Sraf #pragma weak _mutex_unlock = mutex_unlock
24790Sstevel@tonic-gate int
mutex_unlock(mutex_t * mp)24806812Sraf mutex_unlock(mutex_t *mp)
24810Sstevel@tonic-gate {
24820Sstevel@tonic-gate ulwp_t *self = curthread;
24836247Sraf int mtype = mp->mutex_type;
24840Sstevel@tonic-gate uberflags_t *gflags;
24850Sstevel@tonic-gate lwpid_t lwpid;
24860Sstevel@tonic-gate short el;
24870Sstevel@tonic-gate
24880Sstevel@tonic-gate /*
24890Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including
24900Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
24910Sstevel@tonic-gate * no error detection, no lock statistics,
24920Sstevel@tonic-gate * and the process has only a single thread.
24930Sstevel@tonic-gate * (Most likely a traditional single-threaded application.)
24940Sstevel@tonic-gate */
24956247Sraf if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
24966247Sraf self->ul_uberdata->uberflags.uf_all) == 0) {
24970Sstevel@tonic-gate if (mtype) {
24980Sstevel@tonic-gate /*
24990Sstevel@tonic-gate * At this point we know that one or both of the
25000Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
25010Sstevel@tonic-gate */
25020Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
25030Sstevel@tonic-gate return (EPERM);
25040Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
25050Sstevel@tonic-gate mp->mutex_rcount--;
25060Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
25070Sstevel@tonic-gate return (0);
25080Sstevel@tonic-gate }
25090Sstevel@tonic-gate }
25100Sstevel@tonic-gate /*
25110Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation.
25120Sstevel@tonic-gate * Also, there can be no waiters.
25130Sstevel@tonic-gate */
25147907SRoger.Faulkner@Sun.COM sigoff(self);
25150Sstevel@tonic-gate mp->mutex_owner = 0;
25160Sstevel@tonic-gate mp->mutex_lockword = 0;
25177907SRoger.Faulkner@Sun.COM sigon(self);
25180Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
25190Sstevel@tonic-gate return (0);
25200Sstevel@tonic-gate }
25210Sstevel@tonic-gate
25220Sstevel@tonic-gate /*
25230Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
25240Sstevel@tonic-gate * no error detection, and no lock statistics.
25250Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
25260Sstevel@tonic-gate */
25270Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) {
25280Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) {
25290Sstevel@tonic-gate fast_unlock:
25305629Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
25310Sstevel@tonic-gate (void) __lwp_unpark(lwpid);
25320Sstevel@tonic-gate preempt(self);
25330Sstevel@tonic-gate }
25340Sstevel@tonic-gate return (0);
25350Sstevel@tonic-gate }
25360Sstevel@tonic-gate if (el) /* error detection or lock statistics */
25370Sstevel@tonic-gate goto slow_unlock;
25380Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
25390Sstevel@tonic-gate /*
25400Sstevel@tonic-gate * At this point we know that one or both of the
25410Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
25420Sstevel@tonic-gate */
25430Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
25440Sstevel@tonic-gate return (EPERM);
25450Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
25460Sstevel@tonic-gate mp->mutex_rcount--;
25470Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
25480Sstevel@tonic-gate return (0);
25490Sstevel@tonic-gate }
25500Sstevel@tonic-gate goto fast_unlock;
25510Sstevel@tonic-gate }
25520Sstevel@tonic-gate if ((mtype &
25530Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
25540Sstevel@tonic-gate /*
25550Sstevel@tonic-gate * At this point we know that zero, one, or both of the
25560Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and
25570Sstevel@tonic-gate * that the USYNC_PROCESS flag is set.
25580Sstevel@tonic-gate */
25590Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp))
25600Sstevel@tonic-gate return (EPERM);
25610Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
25620Sstevel@tonic-gate mp->mutex_rcount--;
25630Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
25640Sstevel@tonic-gate return (0);
25650Sstevel@tonic-gate }
25665629Sraf mutex_unlock_process(mp, 0);
25670Sstevel@tonic-gate return (0);
25680Sstevel@tonic-gate }
25690Sstevel@tonic-gate }
25700Sstevel@tonic-gate
25710Sstevel@tonic-gate /* else do it the long way */
25720Sstevel@tonic-gate slow_unlock:
25734574Sraf return (mutex_unlock_internal(mp, 0));
25740Sstevel@tonic-gate }
25750Sstevel@tonic-gate
25760Sstevel@tonic-gate /*
25770Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions
25780Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions.
25796812Sraf * We replicate a bit of code from mutex_lock() and mutex_unlock()
25800Sstevel@tonic-gate * to make these functions faster since we know that the mutex type
25810Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal
25820Sstevel@tonic-gate * locking can never fail, so we panic if it does.
25830Sstevel@tonic-gate */
25840Sstevel@tonic-gate void
lmutex_lock(mutex_t * mp)25850Sstevel@tonic-gate lmutex_lock(mutex_t *mp)
25860Sstevel@tonic-gate {
25870Sstevel@tonic-gate ulwp_t *self = curthread;
25880Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
25890Sstevel@tonic-gate
25900Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD);
25910Sstevel@tonic-gate
25920Sstevel@tonic-gate enter_critical(self);
25930Sstevel@tonic-gate /*
25940Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread.
25950Sstevel@tonic-gate * (Most likely a traditional single-threaded application.)
25960Sstevel@tonic-gate */
25970Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) {
25980Sstevel@tonic-gate /*
25990Sstevel@tonic-gate * Only one thread exists; the mutex must be free.
26000Sstevel@tonic-gate */
26010Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0);
26020Sstevel@tonic-gate mp->mutex_lockw = LOCKSET;
26030Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
26040Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
26050Sstevel@tonic-gate } else {
26060Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
26070Sstevel@tonic-gate
26080Sstevel@tonic-gate if (!self->ul_schedctl_called)
26090Sstevel@tonic-gate (void) setup_schedctl();
26100Sstevel@tonic-gate
26110Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) {
26120Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
26130Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
26144613Sraf } else if (mutex_trylock_adaptive(mp, 1) != 0) {
26150Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL);
26160Sstevel@tonic-gate }
26170Sstevel@tonic-gate
26180Sstevel@tonic-gate if (msp)
26190Sstevel@tonic-gate record_begin_hold(msp);
26200Sstevel@tonic-gate }
26210Sstevel@tonic-gate }
26220Sstevel@tonic-gate
26230Sstevel@tonic-gate void
lmutex_unlock(mutex_t * mp)26240Sstevel@tonic-gate lmutex_unlock(mutex_t *mp)
26250Sstevel@tonic-gate {
26260Sstevel@tonic-gate ulwp_t *self = curthread;
26270Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
26280Sstevel@tonic-gate
26290Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD);
26300Sstevel@tonic-gate
26310Sstevel@tonic-gate /*
26320Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread.
26330Sstevel@tonic-gate * (Most likely a traditional single-threaded application.)
26340Sstevel@tonic-gate */
26350Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) {
26360Sstevel@tonic-gate /*
26370Sstevel@tonic-gate * Only one thread exists so there can be no waiters.
26380Sstevel@tonic-gate */
26390Sstevel@tonic-gate mp->mutex_owner = 0;
26400Sstevel@tonic-gate mp->mutex_lockword = 0;
26410Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
26420Sstevel@tonic-gate } else {
26430Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
26440Sstevel@tonic-gate lwpid_t lwpid;
26450Sstevel@tonic-gate
26460Sstevel@tonic-gate if (msp)
26470Sstevel@tonic-gate (void) record_hold_time(msp);
26484574Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
26490Sstevel@tonic-gate (void) __lwp_unpark(lwpid);
26500Sstevel@tonic-gate preempt(self);
26510Sstevel@tonic-gate }
26520Sstevel@tonic-gate }
26530Sstevel@tonic-gate exit_critical(self);
26540Sstevel@tonic-gate }
26550Sstevel@tonic-gate
26562248Sraf /*
26572248Sraf * For specialized code in libc, like the asynchronous i/o code,
26582248Sraf * the following sig_*() locking primitives are used in order
26592248Sraf * to make the code asynchronous signal safe. Signals are
26602248Sraf * deferred while locks acquired by these functions are held.
26612248Sraf */
26622248Sraf void
sig_mutex_lock(mutex_t * mp)26632248Sraf sig_mutex_lock(mutex_t *mp)
26642248Sraf {
266510637SRoger.Faulkner@Sun.COM ulwp_t *self = curthread;
266610637SRoger.Faulkner@Sun.COM
266710637SRoger.Faulkner@Sun.COM sigoff(self);
26686515Sraf (void) mutex_lock(mp);
26692248Sraf }
26702248Sraf
26712248Sraf void
sig_mutex_unlock(mutex_t * mp)26722248Sraf sig_mutex_unlock(mutex_t *mp)
26732248Sraf {
267410637SRoger.Faulkner@Sun.COM ulwp_t *self = curthread;
267510637SRoger.Faulkner@Sun.COM
26766515Sraf (void) mutex_unlock(mp);
267710637SRoger.Faulkner@Sun.COM sigon(self);
26782248Sraf }
26792248Sraf
26802248Sraf int
sig_mutex_trylock(mutex_t * mp)26812248Sraf sig_mutex_trylock(mutex_t *mp)
26822248Sraf {
268310637SRoger.Faulkner@Sun.COM ulwp_t *self = curthread;
26842248Sraf int error;
26852248Sraf
268610637SRoger.Faulkner@Sun.COM sigoff(self);
26876515Sraf if ((error = mutex_trylock(mp)) != 0)
268810637SRoger.Faulkner@Sun.COM sigon(self);
26892248Sraf return (error);
26902248Sraf }
26912248Sraf
26922248Sraf /*
26932248Sraf * sig_cond_wait() is a cancellation point.
26942248Sraf */
26952248Sraf int
sig_cond_wait(cond_t * cv,mutex_t * mp)26962248Sraf sig_cond_wait(cond_t *cv, mutex_t *mp)
26972248Sraf {
26982248Sraf int error;
26992248Sraf
27002248Sraf ASSERT(curthread->ul_sigdefer != 0);
27016515Sraf pthread_testcancel();
27025891Sraf error = __cond_wait(cv, mp);
27032248Sraf if (error == EINTR && curthread->ul_cursig) {
27042248Sraf sig_mutex_unlock(mp);
27052248Sraf /* take the deferred signal here */
27062248Sraf sig_mutex_lock(mp);
27072248Sraf }
27086515Sraf pthread_testcancel();
27092248Sraf return (error);
27102248Sraf }
27112248Sraf
27122248Sraf /*
27132248Sraf * sig_cond_reltimedwait() is a cancellation point.
27142248Sraf */
27152248Sraf int
sig_cond_reltimedwait(cond_t * cv,mutex_t * mp,const timespec_t * ts)27162248Sraf sig_cond_reltimedwait(cond_t *cv, mutex_t *mp, const timespec_t *ts)
27172248Sraf {
27182248Sraf int error;
27192248Sraf
27202248Sraf ASSERT(curthread->ul_sigdefer != 0);
27216515Sraf pthread_testcancel();
27225891Sraf error = __cond_reltimedwait(cv, mp, ts);
27232248Sraf if (error == EINTR && curthread->ul_cursig) {
27242248Sraf sig_mutex_unlock(mp);
27252248Sraf /* take the deferred signal here */
27262248Sraf sig_mutex_lock(mp);
27272248Sraf }
27286515Sraf pthread_testcancel();
27292248Sraf return (error);
27302248Sraf }
27312248Sraf
27325891Sraf /*
27335891Sraf * For specialized code in libc, like the stdio code.
27345891Sraf * the following cancel_safe_*() locking primitives are used in
27355891Sraf * order to make the code cancellation-safe. Cancellation is
27365891Sraf * deferred while locks acquired by these functions are held.
27375891Sraf */
27385891Sraf void
cancel_safe_mutex_lock(mutex_t * mp)27395891Sraf cancel_safe_mutex_lock(mutex_t *mp)
27405891Sraf {
27416515Sraf (void) mutex_lock(mp);
27425891Sraf curthread->ul_libc_locks++;
27435891Sraf }
27445891Sraf
27455891Sraf int
cancel_safe_mutex_trylock(mutex_t * mp)27465891Sraf cancel_safe_mutex_trylock(mutex_t *mp)
27475891Sraf {
27485891Sraf int error;
27495891Sraf
27506515Sraf if ((error = mutex_trylock(mp)) == 0)
27515891Sraf curthread->ul_libc_locks++;
27525891Sraf return (error);
27535891Sraf }
27545891Sraf
27555891Sraf void
cancel_safe_mutex_unlock(mutex_t * mp)27565891Sraf cancel_safe_mutex_unlock(mutex_t *mp)
27575891Sraf {
27585891Sraf ulwp_t *self = curthread;
27595891Sraf
27605891Sraf ASSERT(self->ul_libc_locks != 0);
27615891Sraf
27626515Sraf (void) mutex_unlock(mp);
27635891Sraf
27645891Sraf /*
27655891Sraf * Decrement the count of locks held by cancel_safe_mutex_lock().
27665891Sraf * If we are then in a position to terminate cleanly and
27675891Sraf * if there is a pending cancellation and cancellation
27685891Sraf * is not disabled and we received EINTR from a recent
27695891Sraf * system call then perform the cancellation action now.
27705891Sraf */
27715891Sraf if (--self->ul_libc_locks == 0 &&
27725891Sraf !(self->ul_vfork | self->ul_nocancel |
27735891Sraf self->ul_critical | self->ul_sigdefer) &&
27745891Sraf cancel_active())
27756812Sraf pthread_exit(PTHREAD_CANCELED);
27765891Sraf }
27775891Sraf
27780Sstevel@tonic-gate static int
shared_mutex_held(mutex_t * mparg)27790Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg)
27800Sstevel@tonic-gate {
27810Sstevel@tonic-gate /*
27824574Sraf * The 'volatile' is necessary to make sure the compiler doesn't
27834574Sraf * reorder the tests of the various components of the mutex.
27844574Sraf * They must be tested in this order:
27854574Sraf * mutex_lockw
27864574Sraf * mutex_owner
27874574Sraf * mutex_ownerpid
27884574Sraf * This relies on the fact that everywhere mutex_lockw is cleared,
27894574Sraf * mutex_owner and mutex_ownerpid are cleared before mutex_lockw
27904574Sraf * is cleared, and that everywhere mutex_lockw is set, mutex_owner
27914574Sraf * and mutex_ownerpid are set after mutex_lockw is set, and that
27924574Sraf * mutex_lockw is set or cleared with a memory barrier.
27930Sstevel@tonic-gate */
27940Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg;
27950Sstevel@tonic-gate ulwp_t *self = curthread;
27960Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
27970Sstevel@tonic-gate
27984574Sraf return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid);
27990Sstevel@tonic-gate }
28000Sstevel@tonic-gate
28016812Sraf #pragma weak _mutex_held = mutex_held
28020Sstevel@tonic-gate int
mutex_held(mutex_t * mparg)28036812Sraf mutex_held(mutex_t *mparg)
28040Sstevel@tonic-gate {
28054574Sraf volatile mutex_t *mp = (volatile mutex_t *)mparg;
28064574Sraf
28074574Sraf if (mparg->mutex_type & USYNC_PROCESS)
28084574Sraf return (shared_mutex_held(mparg));
28090Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread));
28100Sstevel@tonic-gate }
28110Sstevel@tonic-gate
28126812Sraf #pragma weak pthread_mutex_destroy = mutex_destroy
28136812Sraf #pragma weak _mutex_destroy = mutex_destroy
28140Sstevel@tonic-gate int
mutex_destroy(mutex_t * mp)28156812Sraf mutex_destroy(mutex_t *mp)
28160Sstevel@tonic-gate {
28174574Sraf if (mp->mutex_type & USYNC_PROCESS)
28184574Sraf forget_lock(mp);
28196515Sraf (void) memset(mp, 0, sizeof (*mp));
28200Sstevel@tonic-gate tdb_sync_obj_deregister(mp);
28210Sstevel@tonic-gate return (0);
28220Sstevel@tonic-gate }
28230Sstevel@tonic-gate
28246812Sraf #pragma weak pthread_mutex_consistent_np = mutex_consistent
28258036SRoger.Faulkner@Sun.COM #pragma weak pthread_mutex_consistent = mutex_consistent
28264574Sraf int
mutex_consistent(mutex_t * mp)28276812Sraf mutex_consistent(mutex_t *mp)
28284574Sraf {
28294574Sraf /*
28304574Sraf * Do this only for an inconsistent, initialized robust lock
28314574Sraf * that we hold. For all other cases, return EINVAL.
28324574Sraf */
28336812Sraf if (mutex_held(mp) &&
28344574Sraf (mp->mutex_type & LOCK_ROBUST) &&
28354574Sraf (mp->mutex_flag & LOCK_INITED) &&
28364574Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
28374574Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
28384574Sraf mp->mutex_rcount = 0;
28394574Sraf return (0);
28404574Sraf }
28414574Sraf return (EINVAL);
28424574Sraf }
28434574Sraf
28440Sstevel@tonic-gate /*
28450Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes,
28460Sstevel@tonic-gate * but we use the same data structure for them.
28470Sstevel@tonic-gate */
28480Sstevel@tonic-gate
28490Sstevel@tonic-gate int
pthread_spin_init(pthread_spinlock_t * lock,int pshared)28506812Sraf pthread_spin_init(pthread_spinlock_t *lock, int pshared)
28510Sstevel@tonic-gate {
28520Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock;
28530Sstevel@tonic-gate
28546515Sraf (void) memset(mp, 0, sizeof (*mp));
28550Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED)
28560Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS;
28570Sstevel@tonic-gate else
28580Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD;
28590Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED;
28600Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC;
28617255Sraf
28627255Sraf /*
28637255Sraf * This should be at the beginning of the function,
28647255Sraf * but for the sake of old broken applications that
28657255Sraf * do not have proper alignment for their mutexes
28667255Sraf * (and don't check the return code from pthread_spin_init),
28677255Sraf * we put it here, after initializing the mutex regardless.
28687255Sraf */
28697255Sraf if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
28707255Sraf curthread->ul_misaligned == 0)
28717255Sraf return (EINVAL);
28727255Sraf
28730Sstevel@tonic-gate return (0);
28740Sstevel@tonic-gate }
28750Sstevel@tonic-gate
28760Sstevel@tonic-gate int
pthread_spin_destroy(pthread_spinlock_t * lock)28776812Sraf pthread_spin_destroy(pthread_spinlock_t *lock)
28780Sstevel@tonic-gate {
28796515Sraf (void) memset(lock, 0, sizeof (*lock));
28800Sstevel@tonic-gate return (0);
28810Sstevel@tonic-gate }
28820Sstevel@tonic-gate
28830Sstevel@tonic-gate int
pthread_spin_trylock(pthread_spinlock_t * lock)28846812Sraf pthread_spin_trylock(pthread_spinlock_t *lock)
28850Sstevel@tonic-gate {
28860Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock;
28870Sstevel@tonic-gate ulwp_t *self = curthread;
28880Sstevel@tonic-gate int error = 0;
28890Sstevel@tonic-gate
28900Sstevel@tonic-gate no_preempt(self);
28910Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0)
28920Sstevel@tonic-gate error = EBUSY;
28930Sstevel@tonic-gate else {
28940Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self;
28950Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS)
28960Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid;
28970Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
28980Sstevel@tonic-gate }
28990Sstevel@tonic-gate preempt(self);
29000Sstevel@tonic-gate return (error);
29010Sstevel@tonic-gate }
29020Sstevel@tonic-gate
29030Sstevel@tonic-gate int
pthread_spin_lock(pthread_spinlock_t * lock)29046812Sraf pthread_spin_lock(pthread_spinlock_t *lock)
29050Sstevel@tonic-gate {
29064574Sraf mutex_t *mp = (mutex_t *)lock;
29074574Sraf ulwp_t *self = curthread;
29084574Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
29094574Sraf int count = 0;
29104574Sraf
29114574Sraf ASSERT(!self->ul_critical || self->ul_bindflags);
29124574Sraf
29134574Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp);
29144574Sraf
29150Sstevel@tonic-gate /*
29160Sstevel@tonic-gate * We don't care whether the owner is running on a processor.
29170Sstevel@tonic-gate * We just spin because that's what this interface requires.
29180Sstevel@tonic-gate */
29190Sstevel@tonic-gate for (;;) {
29200Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */
29214574Sraf no_preempt(self);
29224574Sraf if (set_lock_byte(lockp) == 0)
29234574Sraf break;
29244574Sraf preempt(self);
29250Sstevel@tonic-gate }
29265629Sraf if (count < INT_MAX)
29275629Sraf count++;
29280Sstevel@tonic-gate SMT_PAUSE();
29290Sstevel@tonic-gate }
29304574Sraf mp->mutex_owner = (uintptr_t)self;
29314574Sraf if (mp->mutex_type == USYNC_PROCESS)
29324574Sraf mp->mutex_ownerpid = self->ul_uberdata->pid;
29334574Sraf preempt(self);
29345629Sraf if (count) {
29359397SJonathan.Haslam@Sun.COM DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
29365629Sraf }
29374574Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
29384574Sraf return (0);
29390Sstevel@tonic-gate }
29400Sstevel@tonic-gate
29410Sstevel@tonic-gate int
pthread_spin_unlock(pthread_spinlock_t * lock)29426812Sraf pthread_spin_unlock(pthread_spinlock_t *lock)
29430Sstevel@tonic-gate {
29440Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock;
29450Sstevel@tonic-gate ulwp_t *self = curthread;
29460Sstevel@tonic-gate
29470Sstevel@tonic-gate no_preempt(self);
29480Sstevel@tonic-gate mp->mutex_owner = 0;
29490Sstevel@tonic-gate mp->mutex_ownerpid = 0;
29500Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
29514570Sraf (void) atomic_swap_32(&mp->mutex_lockword, 0);
29520Sstevel@tonic-gate preempt(self);
29530Sstevel@tonic-gate return (0);
29540Sstevel@tonic-gate }
29550Sstevel@tonic-gate
29565629Sraf #define INITIAL_LOCKS 8 /* initial size of ul_heldlocks.array */
29574574Sraf
29584574Sraf /*
29594574Sraf * Find/allocate an entry for 'lock' in our array of held locks.
29604574Sraf */
29614574Sraf static mutex_t **
find_lock_entry(mutex_t * lock)29624574Sraf find_lock_entry(mutex_t *lock)
29634574Sraf {
29644574Sraf ulwp_t *self = curthread;
29654574Sraf mutex_t **remembered = NULL;
29664574Sraf mutex_t **lockptr;
29674574Sraf uint_t nlocks;
29684574Sraf
29694574Sraf if ((nlocks = self->ul_heldlockcnt) != 0)
29704574Sraf lockptr = self->ul_heldlocks.array;
29714574Sraf else {
29724574Sraf nlocks = 1;
29734574Sraf lockptr = &self->ul_heldlocks.single;
29744574Sraf }
29754574Sraf
29764574Sraf for (; nlocks; nlocks--, lockptr++) {
29774574Sraf if (*lockptr == lock)
29784574Sraf return (lockptr);
29794574Sraf if (*lockptr == NULL && remembered == NULL)
29804574Sraf remembered = lockptr;
29814574Sraf }
29824574Sraf if (remembered != NULL) {
29834574Sraf *remembered = lock;
29844574Sraf return (remembered);
29854574Sraf }
29864574Sraf
29874574Sraf /*
29884574Sraf * No entry available. Allocate more space, converting
29894574Sraf * the single entry into an array of entries if necessary.
29904574Sraf */
29914574Sraf if ((nlocks = self->ul_heldlockcnt) == 0) {
29924574Sraf /*
29934574Sraf * Initial allocation of the array.
29944574Sraf * Convert the single entry into an array.
29954574Sraf */
29964574Sraf self->ul_heldlockcnt = nlocks = INITIAL_LOCKS;
29974574Sraf lockptr = lmalloc(nlocks * sizeof (mutex_t *));
29984574Sraf /*
29994574Sraf * The single entry becomes the first entry in the array.
30004574Sraf */
30014574Sraf *lockptr = self->ul_heldlocks.single;
30024574Sraf self->ul_heldlocks.array = lockptr;
30034574Sraf /*
30044574Sraf * Return the next available entry in the array.
30054574Sraf */
30064574Sraf *++lockptr = lock;
30074574Sraf return (lockptr);
30084574Sraf }
30094574Sraf /*
30104574Sraf * Reallocate the array, double the size each time.
30114574Sraf */
30124574Sraf lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *));
30136515Sraf (void) memcpy(lockptr, self->ul_heldlocks.array,
30144574Sraf nlocks * sizeof (mutex_t *));
30154574Sraf lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
30164574Sraf self->ul_heldlocks.array = lockptr;
30174574Sraf self->ul_heldlockcnt *= 2;
30184574Sraf /*
30194574Sraf * Return the next available entry in the newly allocated array.
30204574Sraf */
30214574Sraf *(lockptr += nlocks) = lock;
30224574Sraf return (lockptr);
30234574Sraf }
30244574Sraf
30254574Sraf /*
30264574Sraf * Insert 'lock' into our list of held locks.
30274574Sraf * Currently only used for LOCK_ROBUST mutexes.
30284574Sraf */
30294574Sraf void
remember_lock(mutex_t * lock)30304574Sraf remember_lock(mutex_t *lock)
30314574Sraf {
30324574Sraf (void) find_lock_entry(lock);
30334574Sraf }
30344574Sraf
30354574Sraf /*
30364574Sraf * Remove 'lock' from our list of held locks.
30374574Sraf * Currently only used for LOCK_ROBUST mutexes.
30384574Sraf */
30394574Sraf void
forget_lock(mutex_t * lock)30404574Sraf forget_lock(mutex_t *lock)
30414574Sraf {
30424574Sraf *find_lock_entry(lock) = NULL;
30434574Sraf }
30444574Sraf
30454574Sraf /*
30464574Sraf * Free the array of held locks.
30474574Sraf */
30484574Sraf void
heldlock_free(ulwp_t * ulwp)30494574Sraf heldlock_free(ulwp_t *ulwp)
30504574Sraf {
30514574Sraf uint_t nlocks;
30524574Sraf
30534574Sraf if ((nlocks = ulwp->ul_heldlockcnt) != 0)
30544574Sraf lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
30554574Sraf ulwp->ul_heldlockcnt = 0;
30564574Sraf ulwp->ul_heldlocks.array = NULL;
30574574Sraf }
30584574Sraf
30594574Sraf /*
30604574Sraf * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD.
30614574Sraf * Called from _thrp_exit() to deal with abandoned locks.
30624574Sraf */
30634574Sraf void
heldlock_exit(void)30644574Sraf heldlock_exit(void)
30654574Sraf {
30664574Sraf ulwp_t *self = curthread;
30674574Sraf mutex_t **lockptr;
30684574Sraf uint_t nlocks;
30694574Sraf mutex_t *mp;
30704574Sraf
30714574Sraf if ((nlocks = self->ul_heldlockcnt) != 0)
30724574Sraf lockptr = self->ul_heldlocks.array;
30734574Sraf else {
30744574Sraf nlocks = 1;
30754574Sraf lockptr = &self->ul_heldlocks.single;
30764574Sraf }
30774574Sraf
30784574Sraf for (; nlocks; nlocks--, lockptr++) {
30794574Sraf /*
30804574Sraf * The kernel takes care of transitioning held
30814574Sraf * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD.
30824574Sraf * We avoid that case here.
30834574Sraf */
30844574Sraf if ((mp = *lockptr) != NULL &&
30856812Sraf mutex_held(mp) &&
30864574Sraf (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) ==
30874574Sraf LOCK_ROBUST) {
30884574Sraf mp->mutex_rcount = 0;
30894574Sraf if (!(mp->mutex_flag & LOCK_UNMAPPED))
30904574Sraf mp->mutex_flag |= LOCK_OWNERDEAD;
30914574Sraf (void) mutex_unlock_internal(mp, 1);
30924574Sraf }
30934574Sraf }
30944574Sraf
30954574Sraf heldlock_free(self);
30964574Sraf }
30974574Sraf
30986812Sraf #pragma weak _cond_init = cond_init
30990Sstevel@tonic-gate /* ARGSUSED2 */
31000Sstevel@tonic-gate int
cond_init(cond_t * cvp,int type,void * arg)31016812Sraf cond_init(cond_t *cvp, int type, void *arg)
31020Sstevel@tonic-gate {
31030Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS)
31040Sstevel@tonic-gate return (EINVAL);
31056515Sraf (void) memset(cvp, 0, sizeof (*cvp));
31060Sstevel@tonic-gate cvp->cond_type = (uint16_t)type;
31070Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC;
31087255Sraf
31097255Sraf /*
31107255Sraf * This should be at the beginning of the function,
31117255Sraf * but for the sake of old broken applications that
31127255Sraf * do not have proper alignment for their condvars
31137255Sraf * (and don't check the return code from cond_init),
31147255Sraf * we put it here, after initializing the condvar regardless.
31157255Sraf */
31167255Sraf if (((uintptr_t)cvp & (_LONG_LONG_ALIGNMENT - 1)) &&
31177255Sraf curthread->ul_misaligned == 0)
31187255Sraf return (EINVAL);
31197255Sraf
31200Sstevel@tonic-gate return (0);
31210Sstevel@tonic-gate }
31220Sstevel@tonic-gate
31230Sstevel@tonic-gate /*
31240Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue().
31250Sstevel@tonic-gate *
31260Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up
31270Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due
31280Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply
31290Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()).
31300Sstevel@tonic-gate *
31310Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning.
31320Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue().
31330Sstevel@tonic-gate */
31344574Sraf static int
cond_sleep_queue(cond_t * cvp,mutex_t * mp,timespec_t * tsp)31350Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
31360Sstevel@tonic-gate {
31370Sstevel@tonic-gate ulwp_t *self = curthread;
31380Sstevel@tonic-gate queue_head_t *qp;
31390Sstevel@tonic-gate queue_head_t *mqp;
31400Sstevel@tonic-gate lwpid_t lwpid;
31410Sstevel@tonic-gate int signalled;
31420Sstevel@tonic-gate int error;
31436247Sraf int cv_wake;
31444574Sraf int release_all;
31450Sstevel@tonic-gate
31460Sstevel@tonic-gate /*
31470Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then
31480Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex.
31490Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the
31500Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation.
31510Sstevel@tonic-gate */
31520Sstevel@tonic-gate self->ul_sp = stkptr();
31530Sstevel@tonic-gate qp = queue_lock(cvp, CV);
31546247Sraf enqueue(qp, self, 0);
31550Sstevel@tonic-gate cvp->cond_waiters_user = 1;
31560Sstevel@tonic-gate self->ul_cvmutex = mp;
31576247Sraf self->ul_cv_wake = cv_wake = (tsp != NULL);
31580Sstevel@tonic-gate self->ul_signalled = 0;
31594574Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) {
31604574Sraf mp->mutex_flag &= ~LOCK_OWNERDEAD;
31614574Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE;
31624574Sraf }
31634574Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
31644574Sraf lwpid = mutex_unlock_queue(mp, release_all);
31650Sstevel@tonic-gate for (;;) {
31660Sstevel@tonic-gate set_parking_flag(self, 1);
31670Sstevel@tonic-gate queue_unlock(qp);
31680Sstevel@tonic-gate if (lwpid != 0) {
31690Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid);
31700Sstevel@tonic-gate preempt(self);
31710Sstevel@tonic-gate }
31720Sstevel@tonic-gate /*
31730Sstevel@tonic-gate * We may have a deferred signal present,
31740Sstevel@tonic-gate * in which case we should return EINTR.
31750Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so
31760Sstevel@tonic-gate * and we are cancelable we should return EINTR.
31770Sstevel@tonic-gate * We force an immediate EINTR return from
31780Sstevel@tonic-gate * __lwp_park() by turning our parking flag off.
31790Sstevel@tonic-gate */
31800Sstevel@tonic-gate if (self->ul_cursig != 0 ||
31810Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending))
31820Sstevel@tonic-gate set_parking_flag(self, 0);
31830Sstevel@tonic-gate /*
31840Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp
31850Sstevel@tonic-gate * if we are unparked before the timeout expires.
31860Sstevel@tonic-gate */
31870Sstevel@tonic-gate error = __lwp_park(tsp, lwpid);
31880Sstevel@tonic-gate set_parking_flag(self, 0);
31890Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */
31900Sstevel@tonic-gate /*
31910Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(),
31920Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME),
31930Sstevel@tonic-gate * or we may just have gotten a spurious wakeup.
31940Sstevel@tonic-gate */
31950Sstevel@tonic-gate qp = queue_lock(cvp, CV);
31966247Sraf if (!cv_wake)
31976247Sraf mqp = queue_lock(mp, MX);
31980Sstevel@tonic-gate if (self->ul_sleepq == NULL)
31990Sstevel@tonic-gate break;
32000Sstevel@tonic-gate /*
32010Sstevel@tonic-gate * We are on either the condvar sleep queue or the
32021893Sraf * mutex sleep queue. Break out of the sleep if we
32031893Sraf * were interrupted or we timed out (EINTR or ETIME).
32040Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop.
32050Sstevel@tonic-gate */
32066247Sraf if (!cv_wake && self->ul_sleepq == mqp) { /* mutex queue */
32071893Sraf if (error) {
32086247Sraf mp->mutex_waiters = dequeue_self(mqp);
32091893Sraf break;
32101893Sraf }
32111893Sraf tsp = NULL; /* no more timeout */
32121893Sraf } else if (self->ul_sleepq == qp) { /* condvar queue */
32130Sstevel@tonic-gate if (error) {
32146247Sraf cvp->cond_waiters_user = dequeue_self(qp);
32150Sstevel@tonic-gate break;
32160Sstevel@tonic-gate }
32170Sstevel@tonic-gate /*
32180Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue.
32190Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout.
32200Sstevel@tonic-gate */
32210Sstevel@tonic-gate } else {
32220Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue");
32230Sstevel@tonic-gate }
32246247Sraf if (!cv_wake)
32256247Sraf queue_unlock(mqp);
32260Sstevel@tonic-gate }
32270Sstevel@tonic-gate
32280Sstevel@tonic-gate self->ul_sp = 0;
32296247Sraf self->ul_cv_wake = 0;
32306247Sraf ASSERT(self->ul_cvmutex == NULL);
32310Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
32320Sstevel@tonic-gate self->ul_wchan == NULL);
32330Sstevel@tonic-gate
32340Sstevel@tonic-gate signalled = self->ul_signalled;
32350Sstevel@tonic-gate self->ul_signalled = 0;
32360Sstevel@tonic-gate queue_unlock(qp);
32376247Sraf if (!cv_wake)
32386247Sraf queue_unlock(mqp);
32390Sstevel@tonic-gate
32400Sstevel@tonic-gate /*
32410Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of:
32420Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout,
32430Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it.
32440Sstevel@tonic-gate */
32450Sstevel@tonic-gate if (error && signalled)
32466812Sraf (void) cond_signal(cvp);
32470Sstevel@tonic-gate
32480Sstevel@tonic-gate return (error);
32490Sstevel@tonic-gate }
32500Sstevel@tonic-gate
32517255Sraf static void
cond_wait_check_alignment(cond_t * cvp,mutex_t * mp)32527255Sraf cond_wait_check_alignment(cond_t *cvp, mutex_t *mp)
32537255Sraf {
32547255Sraf if ((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1))
32557255Sraf lock_error(mp, "cond_wait", cvp, "mutex is misaligned");
32567255Sraf if ((uintptr_t)cvp & (_LONG_LONG_ALIGNMENT - 1))
32577255Sraf lock_error(mp, "cond_wait", cvp, "condvar is misaligned");
32587255Sraf }
32597255Sraf
32600Sstevel@tonic-gate int
cond_wait_queue(cond_t * cvp,mutex_t * mp,timespec_t * tsp)32615629Sraf cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
32620Sstevel@tonic-gate {
32630Sstevel@tonic-gate ulwp_t *self = curthread;
32640Sstevel@tonic-gate int error;
32654574Sraf int merror;
32660Sstevel@tonic-gate
32677255Sraf if (self->ul_error_detection && self->ul_misaligned == 0)
32687255Sraf cond_wait_check_alignment(cvp, mp);
32697255Sraf
32700Sstevel@tonic-gate /*
32710Sstevel@tonic-gate * The old thread library was programmed to defer signals
32720Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would
32730Sstevel@tonic-gate * be guaranteed to be held when the application signal
32740Sstevel@tonic-gate * handler was invoked.
32750Sstevel@tonic-gate *
32760Sstevel@tonic-gate * We do not behave this way by default; the state of the
32770Sstevel@tonic-gate * associated mutex in the signal handler is undefined.
32780Sstevel@tonic-gate *
32790Sstevel@tonic-gate * To accommodate applications that depend on the old
32800Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment
32810Sstevel@tonic-gate * variable can be set to 1 and we will behave in the
32820Sstevel@tonic-gate * old way with respect to cond_wait().
32830Sstevel@tonic-gate */
32840Sstevel@tonic-gate if (self->ul_cond_wait_defer)
32850Sstevel@tonic-gate sigoff(self);
32860Sstevel@tonic-gate
32870Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp);
32880Sstevel@tonic-gate
32890Sstevel@tonic-gate /*
32900Sstevel@tonic-gate * Reacquire the mutex.
32910Sstevel@tonic-gate */
32925629Sraf if ((merror = mutex_lock_impl(mp, NULL)) != 0)
32934574Sraf error = merror;
32940Sstevel@tonic-gate
32950Sstevel@tonic-gate /*
32960Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex.
32970Sstevel@tonic-gate */
32980Sstevel@tonic-gate if (self->ul_cond_wait_defer)
32990Sstevel@tonic-gate sigon(self);
33000Sstevel@tonic-gate
33010Sstevel@tonic-gate return (error);
33020Sstevel@tonic-gate }
33030Sstevel@tonic-gate
33040Sstevel@tonic-gate /*
33050Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel().
33060Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above.
33070Sstevel@tonic-gate */
33084574Sraf static int
cond_sleep_kernel(cond_t * cvp,mutex_t * mp,timespec_t * tsp)33090Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
33100Sstevel@tonic-gate {
33110Sstevel@tonic-gate int mtype = mp->mutex_type;
33120Sstevel@tonic-gate ulwp_t *self = curthread;
33130Sstevel@tonic-gate int error;
33140Sstevel@tonic-gate
33154574Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
33164574Sraf _ceil_prio_waive();
33170Sstevel@tonic-gate
33180Sstevel@tonic-gate self->ul_sp = stkptr();
33190Sstevel@tonic-gate self->ul_wchan = cvp;
33207907SRoger.Faulkner@Sun.COM sigoff(self);
33210Sstevel@tonic-gate mp->mutex_owner = 0;
33226057Sraf /* mp->mutex_ownerpid is cleared by ___lwp_cond_wait() */
33236247Sraf if (mtype & LOCK_PRIO_INHERIT) {
33240Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR;
33256247Sraf self->ul_pilocks--;
33266247Sraf }
33270Sstevel@tonic-gate /*
33280Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if
33290Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it
33300Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this
33310Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't
33320Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked
33330Sstevel@tonic-gate * due to taking a deferred signal before going to sleep.
33340Sstevel@tonic-gate */
33350Sstevel@tonic-gate set_parking_flag(self, 1);
33360Sstevel@tonic-gate if (self->ul_cursig != 0 ||
33370Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending))
33380Sstevel@tonic-gate set_parking_flag(self, 0);
33390Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1);
33400Sstevel@tonic-gate set_parking_flag(self, 0);
33417907SRoger.Faulkner@Sun.COM sigon(self);
33420Sstevel@tonic-gate self->ul_sp = 0;
33430Sstevel@tonic-gate self->ul_wchan = NULL;
33440Sstevel@tonic-gate return (error);
33450Sstevel@tonic-gate }
33460Sstevel@tonic-gate
33470Sstevel@tonic-gate int
cond_wait_kernel(cond_t * cvp,mutex_t * mp,timespec_t * tsp)33480Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
33490Sstevel@tonic-gate {
33500Sstevel@tonic-gate ulwp_t *self = curthread;
33510Sstevel@tonic-gate int error;
33520Sstevel@tonic-gate int merror;
33530Sstevel@tonic-gate
33547255Sraf if (self->ul_error_detection && self->ul_misaligned == 0)
33557255Sraf cond_wait_check_alignment(cvp, mp);
33567255Sraf
33570Sstevel@tonic-gate /*
33580Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above.
33590Sstevel@tonic-gate */
33600Sstevel@tonic-gate if (self->ul_cond_wait_defer)
33610Sstevel@tonic-gate sigoff(self);
33620Sstevel@tonic-gate
33630Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp);
33640Sstevel@tonic-gate
33650Sstevel@tonic-gate /*
33660Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait()
33670Sstevel@tonic-gate * with any non-zero return code from mutex_lock().
33680Sstevel@tonic-gate * This addresses robust lock failures in particular;
33690Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE
33700Sstevel@tonic-gate * errors in order to take corrective action.
33710Sstevel@tonic-gate */
33725629Sraf if ((merror = mutex_lock_impl(mp, NULL)) != 0)
33730Sstevel@tonic-gate error = merror;
33740Sstevel@tonic-gate
33750Sstevel@tonic-gate /*
33760Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex.
33770Sstevel@tonic-gate */
33780Sstevel@tonic-gate if (self->ul_cond_wait_defer)
33790Sstevel@tonic-gate sigon(self);
33800Sstevel@tonic-gate
33810Sstevel@tonic-gate return (error);
33820Sstevel@tonic-gate }
33830Sstevel@tonic-gate
33840Sstevel@tonic-gate /*
33856812Sraf * Common code for cond_wait() and cond_timedwait()
33860Sstevel@tonic-gate */
33870Sstevel@tonic-gate int
cond_wait_common(cond_t * cvp,mutex_t * mp,timespec_t * tsp)33880Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
33890Sstevel@tonic-gate {
33900Sstevel@tonic-gate int mtype = mp->mutex_type;
33910Sstevel@tonic-gate hrtime_t begin_sleep = 0;
33920Sstevel@tonic-gate ulwp_t *self = curthread;
33930Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
33940Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
33950Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
33960Sstevel@tonic-gate uint8_t rcount;
33970Sstevel@tonic-gate int error = 0;
33980Sstevel@tonic-gate
33990Sstevel@tonic-gate /*
34000Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states:
34010Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks
34020Sstevel@tonic-gate * shall act as if they were performed immediately at the
34030Sstevel@tonic-gate * beginning of processing for the function and shall cause
34040Sstevel@tonic-gate * an error return, in effect, prior to modifying the state
34050Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable
34060Sstevel@tonic-gate * specified by cond.
34070Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid.
34080Sstevel@tonic-gate */
34090Sstevel@tonic-gate if (tsp != NULL &&
34100Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC))
34110Sstevel@tonic-gate return (EINVAL);
34120Sstevel@tonic-gate
34130Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) {
34140Sstevel@tonic-gate self->ul_sp = stkptr();
34150Sstevel@tonic-gate self->ul_wchan = cvp;
34160Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP;
34170Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp;
34180Sstevel@tonic-gate tdb_event(TD_SLEEP, udp);
34190Sstevel@tonic-gate self->ul_sp = 0;
34200Sstevel@tonic-gate }
34210Sstevel@tonic-gate if (csp) {
34220Sstevel@tonic-gate if (tsp)
34230Sstevel@tonic-gate tdb_incr(csp->cond_timedwait);
34240Sstevel@tonic-gate else
34250Sstevel@tonic-gate tdb_incr(csp->cond_wait);
34260Sstevel@tonic-gate }
34270Sstevel@tonic-gate if (msp)
34280Sstevel@tonic-gate begin_sleep = record_hold_time(msp);
34290Sstevel@tonic-gate else if (csp)
34300Sstevel@tonic-gate begin_sleep = gethrtime();
34310Sstevel@tonic-gate
34320Sstevel@tonic-gate if (self->ul_error_detection) {
34336812Sraf if (!mutex_held(mp))
34340Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL);
34350Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0)
34360Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait",
34375629Sraf cvp, NULL);
34380Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) {
34394574Sraf if (!(mtype & USYNC_PROCESS))
34400Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp,
34415629Sraf "condvar process-shared, "
34425629Sraf "mutex process-private");
34430Sstevel@tonic-gate } else {
34444574Sraf if (mtype & USYNC_PROCESS)
34450Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp,
34465629Sraf "condvar process-private, "
34475629Sraf "mutex process-shared");
34480Sstevel@tonic-gate }
34490Sstevel@tonic-gate }
34500Sstevel@tonic-gate
34510Sstevel@tonic-gate /*
34520Sstevel@tonic-gate * We deal with recursive mutexes by completely
34530Sstevel@tonic-gate * dropping the lock and restoring the recursion
34540Sstevel@tonic-gate * count after waking up. This is arguably wrong,
34550Sstevel@tonic-gate * but it obeys the principle of least astonishment.
34560Sstevel@tonic-gate */
34570Sstevel@tonic-gate rcount = mp->mutex_rcount;
34580Sstevel@tonic-gate mp->mutex_rcount = 0;
34594574Sraf if ((mtype &
34604574Sraf (USYNC_PROCESS | LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) |
34610Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS))
34620Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp);
34630Sstevel@tonic-gate else
34645629Sraf error = cond_wait_queue(cvp, mp, tsp);
34650Sstevel@tonic-gate mp->mutex_rcount = rcount;
34660Sstevel@tonic-gate
34670Sstevel@tonic-gate if (csp) {
34680Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep;
34690Sstevel@tonic-gate if (tsp == NULL)
34700Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse;
34710Sstevel@tonic-gate else {
34720Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse;
34730Sstevel@tonic-gate if (error == ETIME)
34740Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout);
34750Sstevel@tonic-gate }
34760Sstevel@tonic-gate }
34770Sstevel@tonic-gate return (error);
34780Sstevel@tonic-gate }
34790Sstevel@tonic-gate
34800Sstevel@tonic-gate /*
34816812Sraf * cond_wait() is a cancellation point but __cond_wait() is not.
34826812Sraf * Internally, libc calls the non-cancellation version.
34835891Sraf * Other libraries need to use pthread_setcancelstate(), as appropriate,
34845891Sraf * since __cond_wait() is not exported from libc.
34850Sstevel@tonic-gate */
34860Sstevel@tonic-gate int
__cond_wait(cond_t * cvp,mutex_t * mp)34875891Sraf __cond_wait(cond_t *cvp, mutex_t *mp)
34880Sstevel@tonic-gate {
34890Sstevel@tonic-gate ulwp_t *self = curthread;
34900Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
34910Sstevel@tonic-gate uberflags_t *gflags;
34920Sstevel@tonic-gate
34938036SRoger.Faulkner@Sun.COM if ((mp->mutex_type & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
34948036SRoger.Faulkner@Sun.COM !mutex_held(mp))
34958036SRoger.Faulkner@Sun.COM return (EPERM);
34968036SRoger.Faulkner@Sun.COM
34970Sstevel@tonic-gate /*
34980Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus
34990Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing.
35000Sstevel@tonic-gate */
35010Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL &&
35020Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted |
35030Sstevel@tonic-gate self->ul_td_events_enable |
35040Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0)
35055629Sraf return (cond_wait_queue(cvp, mp, NULL));
35060Sstevel@tonic-gate
35070Sstevel@tonic-gate /*
35080Sstevel@tonic-gate * Else do it the long way.
35090Sstevel@tonic-gate */
35100Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL));
35110Sstevel@tonic-gate }
35120Sstevel@tonic-gate
35136812Sraf #pragma weak _cond_wait = cond_wait
35140Sstevel@tonic-gate int
cond_wait(cond_t * cvp,mutex_t * mp)35156812Sraf cond_wait(cond_t *cvp, mutex_t *mp)
35160Sstevel@tonic-gate {
35170Sstevel@tonic-gate int error;
35180Sstevel@tonic-gate
35190Sstevel@tonic-gate _cancelon();
35205891Sraf error = __cond_wait(cvp, mp);
35210Sstevel@tonic-gate if (error == EINTR)
35220Sstevel@tonic-gate _canceloff();
35230Sstevel@tonic-gate else
35240Sstevel@tonic-gate _canceloff_nocancel();
35250Sstevel@tonic-gate return (error);
35260Sstevel@tonic-gate }
35270Sstevel@tonic-gate
35285891Sraf /*
35295891Sraf * pthread_cond_wait() is a cancellation point.
35305891Sraf */
35310Sstevel@tonic-gate int
pthread_cond_wait(pthread_cond_t * _RESTRICT_KYWD cvp,pthread_mutex_t * _RESTRICT_KYWD mp)35326812Sraf pthread_cond_wait(pthread_cond_t *_RESTRICT_KYWD cvp,
35336812Sraf pthread_mutex_t *_RESTRICT_KYWD mp)
35340Sstevel@tonic-gate {
35350Sstevel@tonic-gate int error;
35360Sstevel@tonic-gate
35376812Sraf error = cond_wait((cond_t *)cvp, (mutex_t *)mp);
35380Sstevel@tonic-gate return ((error == EINTR)? 0 : error);
35390Sstevel@tonic-gate }
35400Sstevel@tonic-gate
35410Sstevel@tonic-gate /*
35426812Sraf * cond_timedwait() is a cancellation point but __cond_timedwait() is not.
35430Sstevel@tonic-gate */
35440Sstevel@tonic-gate int
__cond_timedwait(cond_t * cvp,mutex_t * mp,const timespec_t * abstime)35455891Sraf __cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
35460Sstevel@tonic-gate {
35470Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid;
35480Sstevel@tonic-gate timespec_t reltime;
35490Sstevel@tonic-gate int error;
35500Sstevel@tonic-gate
35518036SRoger.Faulkner@Sun.COM if ((mp->mutex_type & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
35528036SRoger.Faulkner@Sun.COM !mutex_held(mp))
35538036SRoger.Faulkner@Sun.COM return (EPERM);
35548036SRoger.Faulkner@Sun.COM
35550Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES)
35560Sstevel@tonic-gate clock_id = CLOCK_REALTIME;
35570Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime);
35580Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime);
35590Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) {
35600Sstevel@tonic-gate /*
35610Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout.
35620Sstevel@tonic-gate * This can happen if we return because someone resets
35630Sstevel@tonic-gate * the system clock. Just return zero in this case,
35640Sstevel@tonic-gate * giving a spurious wakeup but not a timeout.
35650Sstevel@tonic-gate */
35660Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC +
35670Sstevel@tonic-gate abstime->tv_nsec > gethrtime())
35680Sstevel@tonic-gate error = 0;
35690Sstevel@tonic-gate }
35700Sstevel@tonic-gate return (error);
35710Sstevel@tonic-gate }
35720Sstevel@tonic-gate
35730Sstevel@tonic-gate int
cond_timedwait(cond_t * cvp,mutex_t * mp,const timespec_t * abstime)35746812Sraf cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
35750Sstevel@tonic-gate {
35760Sstevel@tonic-gate int error;
35770Sstevel@tonic-gate
35780Sstevel@tonic-gate _cancelon();
35795891Sraf error = __cond_timedwait(cvp, mp, abstime);
35800Sstevel@tonic-gate if (error == EINTR)
35810Sstevel@tonic-gate _canceloff();
35820Sstevel@tonic-gate else
35830Sstevel@tonic-gate _canceloff_nocancel();
35840Sstevel@tonic-gate return (error);
35850Sstevel@tonic-gate }
35860Sstevel@tonic-gate
35875891Sraf /*
35885891Sraf * pthread_cond_timedwait() is a cancellation point.
35895891Sraf */
35900Sstevel@tonic-gate int
pthread_cond_timedwait(pthread_cond_t * _RESTRICT_KYWD cvp,pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD abstime)35916812Sraf pthread_cond_timedwait(pthread_cond_t *_RESTRICT_KYWD cvp,
35926812Sraf pthread_mutex_t *_RESTRICT_KYWD mp,
35936812Sraf const struct timespec *_RESTRICT_KYWD abstime)
35940Sstevel@tonic-gate {
35950Sstevel@tonic-gate int error;
35960Sstevel@tonic-gate
35976812Sraf error = cond_timedwait((cond_t *)cvp, (mutex_t *)mp, abstime);
35980Sstevel@tonic-gate if (error == ETIME)
35990Sstevel@tonic-gate error = ETIMEDOUT;
36000Sstevel@tonic-gate else if (error == EINTR)
36010Sstevel@tonic-gate error = 0;
36020Sstevel@tonic-gate return (error);
36030Sstevel@tonic-gate }
36040Sstevel@tonic-gate
36050Sstevel@tonic-gate /*
36066812Sraf * cond_reltimedwait() is a cancellation point but __cond_reltimedwait() is not.
36070Sstevel@tonic-gate */
36080Sstevel@tonic-gate int
__cond_reltimedwait(cond_t * cvp,mutex_t * mp,const timespec_t * reltime)36095891Sraf __cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
36100Sstevel@tonic-gate {
36110Sstevel@tonic-gate timespec_t tslocal = *reltime;
36120Sstevel@tonic-gate
36138036SRoger.Faulkner@Sun.COM if ((mp->mutex_type & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
36148036SRoger.Faulkner@Sun.COM !mutex_held(mp))
36158036SRoger.Faulkner@Sun.COM return (EPERM);
36168036SRoger.Faulkner@Sun.COM
36170Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal));
36180Sstevel@tonic-gate }
36190Sstevel@tonic-gate
36200Sstevel@tonic-gate int
cond_reltimedwait(cond_t * cvp,mutex_t * mp,const timespec_t * reltime)36216812Sraf cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
36220Sstevel@tonic-gate {
36230Sstevel@tonic-gate int error;
36240Sstevel@tonic-gate
36250Sstevel@tonic-gate _cancelon();
36265891Sraf error = __cond_reltimedwait(cvp, mp, reltime);
36270Sstevel@tonic-gate if (error == EINTR)
36280Sstevel@tonic-gate _canceloff();
36290Sstevel@tonic-gate else
36300Sstevel@tonic-gate _canceloff_nocancel();
36310Sstevel@tonic-gate return (error);
36320Sstevel@tonic-gate }
36330Sstevel@tonic-gate
36340Sstevel@tonic-gate int
pthread_cond_reltimedwait_np(pthread_cond_t * _RESTRICT_KYWD cvp,pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD reltime)36356812Sraf pthread_cond_reltimedwait_np(pthread_cond_t *_RESTRICT_KYWD cvp,
36366812Sraf pthread_mutex_t *_RESTRICT_KYWD mp,
36376812Sraf const struct timespec *_RESTRICT_KYWD reltime)
36380Sstevel@tonic-gate {
36390Sstevel@tonic-gate int error;
36400Sstevel@tonic-gate
36416812Sraf error = cond_reltimedwait((cond_t *)cvp, (mutex_t *)mp, reltime);
36420Sstevel@tonic-gate if (error == ETIME)
36430Sstevel@tonic-gate error = ETIMEDOUT;
36440Sstevel@tonic-gate else if (error == EINTR)
36450Sstevel@tonic-gate error = 0;
36460Sstevel@tonic-gate return (error);
36470Sstevel@tonic-gate }
36480Sstevel@tonic-gate
36496812Sraf #pragma weak pthread_cond_signal = cond_signal
36506812Sraf #pragma weak _cond_signal = cond_signal
36510Sstevel@tonic-gate int
cond_signal(cond_t * cvp)36526812Sraf cond_signal(cond_t *cvp)
36530Sstevel@tonic-gate {
36540Sstevel@tonic-gate ulwp_t *self = curthread;
36550Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
36560Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
36570Sstevel@tonic-gate int error = 0;
36586247Sraf int more;
36596247Sraf lwpid_t lwpid;
36600Sstevel@tonic-gate queue_head_t *qp;
36610Sstevel@tonic-gate mutex_t *mp;
36620Sstevel@tonic-gate queue_head_t *mqp;
36630Sstevel@tonic-gate ulwp_t **ulwpp;
36640Sstevel@tonic-gate ulwp_t *ulwp;
36656247Sraf ulwp_t *prev;
36660Sstevel@tonic-gate
36670Sstevel@tonic-gate if (csp)
36680Sstevel@tonic-gate tdb_incr(csp->cond_signal);
36690Sstevel@tonic-gate
36700Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */
36716812Sraf error = _lwp_cond_signal(cvp);
36720Sstevel@tonic-gate
36730Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */
36740Sstevel@tonic-gate return (error);
36750Sstevel@tonic-gate
36760Sstevel@tonic-gate /*
36770Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep
36780Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up.
36790Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire.
36800Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag
36810Sstevel@tonic-gate * is set, just dequeue and unpark him.
36820Sstevel@tonic-gate */
36830Sstevel@tonic-gate qp = queue_lock(cvp, CV);
36846247Sraf ulwpp = queue_slot(qp, &prev, &more);
36856247Sraf cvp->cond_waiters_user = more;
36866247Sraf if (ulwpp == NULL) { /* no one on the sleep queue */
36870Sstevel@tonic-gate queue_unlock(qp);
36880Sstevel@tonic-gate return (error);
36890Sstevel@tonic-gate }
36906247Sraf ulwp = *ulwpp;
36910Sstevel@tonic-gate
36920Sstevel@tonic-gate /*
36930Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal().
36940Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently,
36950Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout.
36960Sstevel@tonic-gate * These latter conditions must not consume a cond_signal().
36970Sstevel@tonic-gate */
36980Sstevel@tonic-gate ulwp->ul_signalled = 1;
36990Sstevel@tonic-gate
37000Sstevel@tonic-gate /*
37010Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL
37020Sstevel@tonic-gate * while we move him to the mutex queue so that he can
37030Sstevel@tonic-gate * deal properly with spurious wakeups.
37040Sstevel@tonic-gate */
37056247Sraf queue_unlink(qp, ulwpp, prev);
37060Sstevel@tonic-gate
37070Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */
37080Sstevel@tonic-gate ulwp->ul_cvmutex = NULL;
37090Sstevel@tonic-gate ASSERT(mp != NULL);
37100Sstevel@tonic-gate
37110Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
37126247Sraf /* just wake him up */
37136247Sraf lwpid = ulwp->ul_lwpid;
37140Sstevel@tonic-gate no_preempt(self);
37150Sstevel@tonic-gate ulwp->ul_sleepq = NULL;
37160Sstevel@tonic-gate ulwp->ul_wchan = NULL;
37170Sstevel@tonic-gate queue_unlock(qp);
37180Sstevel@tonic-gate (void) __lwp_unpark(lwpid);
37190Sstevel@tonic-gate preempt(self);
37200Sstevel@tonic-gate } else {
37216247Sraf /* move him to the mutex queue */
37220Sstevel@tonic-gate mqp = queue_lock(mp, MX);
37236247Sraf enqueue(mqp, ulwp, 0);
37240Sstevel@tonic-gate mp->mutex_waiters = 1;
37250Sstevel@tonic-gate queue_unlock(mqp);
37260Sstevel@tonic-gate queue_unlock(qp);
37270Sstevel@tonic-gate }
37280Sstevel@tonic-gate
37290Sstevel@tonic-gate return (error);
37300Sstevel@tonic-gate }
37310Sstevel@tonic-gate
37324570Sraf /*
37334574Sraf * Utility function called by mutex_wakeup_all(), cond_broadcast(),
37344574Sraf * and rw_queue_release() to (re)allocate a big buffer to hold the
37354574Sraf * lwpids of all the threads to be set running after they are removed
37364574Sraf * from their sleep queues. Since we are holding a queue lock, we
37374574Sraf * cannot call any function that might acquire a lock. mmap(), munmap(),
37384574Sraf * lwp_unpark_all() are simple system calls and are safe in this regard.
37394570Sraf */
37404570Sraf lwpid_t *
alloc_lwpids(lwpid_t * lwpid,int * nlwpid_ptr,int * maxlwps_ptr)37414570Sraf alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr)
37424570Sraf {
37434570Sraf /*
37444570Sraf * Allocate NEWLWPS ids on the first overflow.
37454570Sraf * Double the allocation each time after that.
37464570Sraf */
37474570Sraf int nlwpid = *nlwpid_ptr;
37484570Sraf int maxlwps = *maxlwps_ptr;
37494570Sraf int first_allocation;
37504570Sraf int newlwps;
37514570Sraf void *vaddr;
37524570Sraf
37534570Sraf ASSERT(nlwpid == maxlwps);
37544570Sraf
37554570Sraf first_allocation = (maxlwps == MAXLWPS);
37564570Sraf newlwps = first_allocation? NEWLWPS : 2 * maxlwps;
37576515Sraf vaddr = mmap(NULL, newlwps * sizeof (lwpid_t),
37584570Sraf PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0);
37594570Sraf
37604570Sraf if (vaddr == MAP_FAILED) {
37614570Sraf /*
37624570Sraf * Let's hope this never happens.
37634570Sraf * If it does, then we have a terrible
37644570Sraf * thundering herd on our hands.
37654570Sraf */
37664570Sraf (void) __lwp_unpark_all(lwpid, nlwpid);
37674570Sraf *nlwpid_ptr = 0;
37684570Sraf } else {
37696515Sraf (void) memcpy(vaddr, lwpid, maxlwps * sizeof (lwpid_t));
37704570Sraf if (!first_allocation)
37716515Sraf (void) munmap((caddr_t)lwpid,
37724570Sraf maxlwps * sizeof (lwpid_t));
37734570Sraf lwpid = vaddr;
37744570Sraf *maxlwps_ptr = newlwps;
37754570Sraf }
37764570Sraf
37774570Sraf return (lwpid);
37784570Sraf }
37790Sstevel@tonic-gate
37806812Sraf #pragma weak pthread_cond_broadcast = cond_broadcast
37816812Sraf #pragma weak _cond_broadcast = cond_broadcast
37820Sstevel@tonic-gate int
cond_broadcast(cond_t * cvp)37836812Sraf cond_broadcast(cond_t *cvp)
37840Sstevel@tonic-gate {
37850Sstevel@tonic-gate ulwp_t *self = curthread;
37860Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
37870Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
37880Sstevel@tonic-gate int error = 0;
37890Sstevel@tonic-gate queue_head_t *qp;
37906247Sraf queue_root_t *qrp;
37910Sstevel@tonic-gate mutex_t *mp;
37920Sstevel@tonic-gate mutex_t *mp_cache = NULL;
37934570Sraf queue_head_t *mqp = NULL;
37940Sstevel@tonic-gate ulwp_t *ulwp;
37954570Sraf int nlwpid = 0;
37964570Sraf int maxlwps = MAXLWPS;
37970Sstevel@tonic-gate lwpid_t buffer[MAXLWPS];
37980Sstevel@tonic-gate lwpid_t *lwpid = buffer;
37990Sstevel@tonic-gate
38000Sstevel@tonic-gate if (csp)
38010Sstevel@tonic-gate tdb_incr(csp->cond_broadcast);
38020Sstevel@tonic-gate
38030Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */
38046812Sraf error = _lwp_cond_broadcast(cvp);
38050Sstevel@tonic-gate
38060Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */
38070Sstevel@tonic-gate return (error);
38080Sstevel@tonic-gate
38090Sstevel@tonic-gate /*
38100Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep
38110Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up.
38120Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire.
38130Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag
38140Sstevel@tonic-gate * is set, just dequeue and unpark them.
38150Sstevel@tonic-gate *
38160Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[].
38170Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after
38180Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep
38190Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our
38200Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call
38210Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow
38220Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca()
38234570Sraf * either because the application may have allocated a small
38244570Sraf * stack and we don't want to overrun the stack. So we call
38254570Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap()
38260Sstevel@tonic-gate * system call directly since that path acquires no locks.
38270Sstevel@tonic-gate */
38280Sstevel@tonic-gate qp = queue_lock(cvp, CV);
38290Sstevel@tonic-gate cvp->cond_waiters_user = 0;
38306247Sraf for (;;) {
38316247Sraf if ((qrp = qp->qh_root) == NULL ||
38326247Sraf (ulwp = qrp->qr_head) == NULL)
38336247Sraf break;
38346247Sraf ASSERT(ulwp->ul_wchan == cvp);
38356247Sraf queue_unlink(qp, &qrp->qr_head, NULL);
38360Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */
38370Sstevel@tonic-gate ulwp->ul_cvmutex = NULL;
38380Sstevel@tonic-gate ASSERT(mp != NULL);
38390Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
38406247Sraf /* just wake him up */
38410Sstevel@tonic-gate ulwp->ul_sleepq = NULL;
38420Sstevel@tonic-gate ulwp->ul_wchan = NULL;
38434570Sraf if (nlwpid == maxlwps)
38444570Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
38450Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid;
38460Sstevel@tonic-gate } else {
38476247Sraf /* move him to the mutex queue */
38480Sstevel@tonic-gate if (mp != mp_cache) {
38490Sstevel@tonic-gate mp_cache = mp;
38504570Sraf if (mqp != NULL)
38514570Sraf queue_unlock(mqp);
38524570Sraf mqp = queue_lock(mp, MX);
38530Sstevel@tonic-gate }
38546247Sraf enqueue(mqp, ulwp, 0);
38550Sstevel@tonic-gate mp->mutex_waiters = 1;
38560Sstevel@tonic-gate }
38570Sstevel@tonic-gate }
38584570Sraf if (mqp != NULL)
38594570Sraf queue_unlock(mqp);
38604570Sraf if (nlwpid == 0) {
38614570Sraf queue_unlock(qp);
38624570Sraf } else {
38634570Sraf no_preempt(self);
38644570Sraf queue_unlock(qp);
38650Sstevel@tonic-gate if (nlwpid == 1)
38660Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]);
38670Sstevel@tonic-gate else
38680Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid);
38694570Sraf preempt(self);
38700Sstevel@tonic-gate }
38710Sstevel@tonic-gate if (lwpid != buffer)
38726515Sraf (void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
38730Sstevel@tonic-gate return (error);
38740Sstevel@tonic-gate }
38750Sstevel@tonic-gate
38766812Sraf #pragma weak pthread_cond_destroy = cond_destroy
38770Sstevel@tonic-gate int
cond_destroy(cond_t * cvp)38786812Sraf cond_destroy(cond_t *cvp)
38790Sstevel@tonic-gate {
38800Sstevel@tonic-gate cvp->cond_magic = 0;
38810Sstevel@tonic-gate tdb_sync_obj_deregister(cvp);
38820Sstevel@tonic-gate return (0);
38830Sstevel@tonic-gate }
38840Sstevel@tonic-gate
38850Sstevel@tonic-gate #if defined(THREAD_DEBUG)
38860Sstevel@tonic-gate void
assert_no_libc_locks_held(void)38870Sstevel@tonic-gate assert_no_libc_locks_held(void)
38880Sstevel@tonic-gate {
38890Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
38900Sstevel@tonic-gate }
38910Sstevel@tonic-gate
38920Sstevel@tonic-gate /* protected by link_lock */
38930Sstevel@tonic-gate uint64_t spin_lock_spin;
38940Sstevel@tonic-gate uint64_t spin_lock_spin2;
38950Sstevel@tonic-gate uint64_t spin_lock_sleep;
38960Sstevel@tonic-gate uint64_t spin_lock_wakeup;
38970Sstevel@tonic-gate
38980Sstevel@tonic-gate /*
38990Sstevel@tonic-gate * Record spin lock statistics.
39000Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit().
39010Sstevel@tonic-gate * Also called via atexit() from the thread calling
39020Sstevel@tonic-gate * exit() to do all the other threads as well.
39030Sstevel@tonic-gate */
39040Sstevel@tonic-gate void
record_spin_locks(ulwp_t * ulwp)39050Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp)
39060Sstevel@tonic-gate {
39070Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin;
39080Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2;
39090Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep;
39100Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup;
39110Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0;
39120Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0;
39130Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0;
39140Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0;
39150Sstevel@tonic-gate }
39160Sstevel@tonic-gate
39170Sstevel@tonic-gate /*
39180Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr.
39190Sstevel@tonic-gate */
39200Sstevel@tonic-gate #include <stdio.h>
39210Sstevel@tonic-gate void
dump_queue_statistics(void)39220Sstevel@tonic-gate dump_queue_statistics(void)
39230Sstevel@tonic-gate {
39240Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
39250Sstevel@tonic-gate queue_head_t *qp;
39260Sstevel@tonic-gate int qn;
39270Sstevel@tonic-gate uint64_t spin_lock_total = 0;
39280Sstevel@tonic-gate
39290Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0)
39300Sstevel@tonic-gate return;
39310Sstevel@tonic-gate
39320Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 ||
39336247Sraf fprintf(stderr, "queue# lockcount max qlen max hlen\n") < 0)
39340Sstevel@tonic-gate return;
39350Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) {
39360Sstevel@tonic-gate if (qp->qh_lockcount == 0)
39370Sstevel@tonic-gate continue;
39380Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount;
39396247Sraf if (fprintf(stderr, "%5d %12llu%12u%12u\n", qn,
39406247Sraf (u_longlong_t)qp->qh_lockcount,
39416247Sraf qp->qh_qmax, qp->qh_hmax) < 0)
39425629Sraf return;
39430Sstevel@tonic-gate }
39440Sstevel@tonic-gate
39450Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 ||
39466247Sraf fprintf(stderr, "queue# lockcount max qlen max hlen\n") < 0)
39470Sstevel@tonic-gate return;
39480Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) {
39490Sstevel@tonic-gate if (qp->qh_lockcount == 0)
39500Sstevel@tonic-gate continue;
39510Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount;
39526247Sraf if (fprintf(stderr, "%5d %12llu%12u%12u\n", qn,
39536247Sraf (u_longlong_t)qp->qh_lockcount,
39546247Sraf qp->qh_qmax, qp->qh_hmax) < 0)
39555629Sraf return;
39560Sstevel@tonic-gate }
39570Sstevel@tonic-gate
39580Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n",
39595629Sraf (u_longlong_t)spin_lock_total);
39600Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n",
39615629Sraf (u_longlong_t)spin_lock_spin);
39620Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n",
39635629Sraf (u_longlong_t)spin_lock_spin2);
39640Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n",
39655629Sraf (u_longlong_t)spin_lock_sleep);
39660Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n",
39675629Sraf (u_longlong_t)spin_lock_wakeup);
39680Sstevel@tonic-gate }
39696247Sraf #endif
3970