1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include "lint.h" 30*0Sstevel@tonic-gate #include "thr_uberdata.h" 31*0Sstevel@tonic-gate 32*0Sstevel@tonic-gate #include <sys/sdt.h> 33*0Sstevel@tonic-gate 34*0Sstevel@tonic-gate #define TRY_FLAG 0x10 35*0Sstevel@tonic-gate #define READ_LOCK 0 36*0Sstevel@tonic-gate #define WRITE_LOCK 1 37*0Sstevel@tonic-gate #define READ_LOCK_TRY (READ_LOCK | TRY_FLAG) 38*0Sstevel@tonic-gate #define WRITE_LOCK_TRY (WRITE_LOCK | TRY_FLAG) 39*0Sstevel@tonic-gate 40*0Sstevel@tonic-gate #define NLOCKS 4 /* initial number of readlock_t structs allocated */ 41*0Sstevel@tonic-gate 42*0Sstevel@tonic-gate /* 43*0Sstevel@tonic-gate * Find/allocate an entry for rwlp in our array of rwlocks held for reading. 44*0Sstevel@tonic-gate */ 45*0Sstevel@tonic-gate static readlock_t * 46*0Sstevel@tonic-gate rwl_entry(rwlock_t *rwlp) 47*0Sstevel@tonic-gate { 48*0Sstevel@tonic-gate ulwp_t *self = curthread; 49*0Sstevel@tonic-gate readlock_t *remembered = NULL; 50*0Sstevel@tonic-gate readlock_t *readlockp; 51*0Sstevel@tonic-gate uint_t nlocks; 52*0Sstevel@tonic-gate 53*0Sstevel@tonic-gate if ((nlocks = self->ul_rdlocks) != 0) 54*0Sstevel@tonic-gate readlockp = self->ul_readlock.array; 55*0Sstevel@tonic-gate else { 56*0Sstevel@tonic-gate nlocks = 1; 57*0Sstevel@tonic-gate readlockp = &self->ul_readlock.single; 58*0Sstevel@tonic-gate } 59*0Sstevel@tonic-gate 60*0Sstevel@tonic-gate for (; nlocks; nlocks--, readlockp++) { 61*0Sstevel@tonic-gate if (readlockp->rd_rwlock == rwlp) 62*0Sstevel@tonic-gate return (readlockp); 63*0Sstevel@tonic-gate if (readlockp->rd_count == 0 && remembered == NULL) 64*0Sstevel@tonic-gate remembered = readlockp; 65*0Sstevel@tonic-gate } 66*0Sstevel@tonic-gate if (remembered != NULL) { 67*0Sstevel@tonic-gate remembered->rd_rwlock = rwlp; 68*0Sstevel@tonic-gate return (remembered); 69*0Sstevel@tonic-gate } 70*0Sstevel@tonic-gate 71*0Sstevel@tonic-gate /* 72*0Sstevel@tonic-gate * No entry available. Allocate more space, converting the single 73*0Sstevel@tonic-gate * readlock_t entry into an array of readlock_t entries if necessary. 74*0Sstevel@tonic-gate */ 75*0Sstevel@tonic-gate if ((nlocks = self->ul_rdlocks) == 0) { 76*0Sstevel@tonic-gate /* 77*0Sstevel@tonic-gate * Initial allocation of the readlock_t array. 78*0Sstevel@tonic-gate * Convert the single entry into an array. 79*0Sstevel@tonic-gate */ 80*0Sstevel@tonic-gate self->ul_rdlocks = nlocks = NLOCKS; 81*0Sstevel@tonic-gate readlockp = lmalloc(nlocks * sizeof (readlock_t)); 82*0Sstevel@tonic-gate /* 83*0Sstevel@tonic-gate * The single readlock_t becomes the first entry in the array. 84*0Sstevel@tonic-gate */ 85*0Sstevel@tonic-gate *readlockp = self->ul_readlock.single; 86*0Sstevel@tonic-gate self->ul_readlock.single.rd_count = 0; 87*0Sstevel@tonic-gate self->ul_readlock.array = readlockp; 88*0Sstevel@tonic-gate /* 89*0Sstevel@tonic-gate * Return the next available entry in the array. 90*0Sstevel@tonic-gate */ 91*0Sstevel@tonic-gate (++readlockp)->rd_rwlock = rwlp; 92*0Sstevel@tonic-gate return (readlockp); 93*0Sstevel@tonic-gate } 94*0Sstevel@tonic-gate /* 95*0Sstevel@tonic-gate * Reallocate the array, double the size each time. 96*0Sstevel@tonic-gate */ 97*0Sstevel@tonic-gate readlockp = lmalloc(nlocks * 2 * sizeof (readlock_t)); 98*0Sstevel@tonic-gate (void) _memcpy(readlockp, self->ul_readlock.array, 99*0Sstevel@tonic-gate nlocks * sizeof (readlock_t)); 100*0Sstevel@tonic-gate lfree(self->ul_readlock.array, nlocks * sizeof (readlock_t)); 101*0Sstevel@tonic-gate self->ul_readlock.array = readlockp; 102*0Sstevel@tonic-gate self->ul_rdlocks *= 2; 103*0Sstevel@tonic-gate /* 104*0Sstevel@tonic-gate * Return the next available entry in the newly allocated array. 105*0Sstevel@tonic-gate */ 106*0Sstevel@tonic-gate (readlockp += nlocks)->rd_rwlock = rwlp; 107*0Sstevel@tonic-gate return (readlockp); 108*0Sstevel@tonic-gate } 109*0Sstevel@tonic-gate 110*0Sstevel@tonic-gate /* 111*0Sstevel@tonic-gate * Free the array of rwlocks held for reading. 112*0Sstevel@tonic-gate */ 113*0Sstevel@tonic-gate void 114*0Sstevel@tonic-gate rwl_free(ulwp_t *ulwp) 115*0Sstevel@tonic-gate { 116*0Sstevel@tonic-gate uint_t nlocks; 117*0Sstevel@tonic-gate 118*0Sstevel@tonic-gate if ((nlocks = ulwp->ul_rdlocks) != 0) 119*0Sstevel@tonic-gate lfree(ulwp->ul_readlock.array, nlocks * sizeof (readlock_t)); 120*0Sstevel@tonic-gate ulwp->ul_rdlocks = 0; 121*0Sstevel@tonic-gate ulwp->ul_readlock.single.rd_rwlock = NULL; 122*0Sstevel@tonic-gate ulwp->ul_readlock.single.rd_count = 0; 123*0Sstevel@tonic-gate } 124*0Sstevel@tonic-gate 125*0Sstevel@tonic-gate /* 126*0Sstevel@tonic-gate * Check if a reader version of the lock is held by the current thread. 127*0Sstevel@tonic-gate * rw_read_is_held() is private to libc. 128*0Sstevel@tonic-gate */ 129*0Sstevel@tonic-gate #pragma weak rw_read_is_held = _rw_read_held 130*0Sstevel@tonic-gate #pragma weak rw_read_held = _rw_read_held 131*0Sstevel@tonic-gate int 132*0Sstevel@tonic-gate _rw_read_held(rwlock_t *rwlp) 133*0Sstevel@tonic-gate { 134*0Sstevel@tonic-gate ulwp_t *self; 135*0Sstevel@tonic-gate readlock_t *readlockp; 136*0Sstevel@tonic-gate uint_t nlocks; 137*0Sstevel@tonic-gate 138*0Sstevel@tonic-gate /* quick answer */ 139*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { 140*0Sstevel@tonic-gate if (!((uint32_t)rwlp->rwlock_readers & URW_READERS_MASK)) 141*0Sstevel@tonic-gate return (0); 142*0Sstevel@tonic-gate } else if (rwlp->rwlock_readers <= 0) { 143*0Sstevel@tonic-gate return (0); 144*0Sstevel@tonic-gate } 145*0Sstevel@tonic-gate 146*0Sstevel@tonic-gate /* 147*0Sstevel@tonic-gate * The lock is held for reading by some thread. 148*0Sstevel@tonic-gate * Search our array of rwlocks held for reading for a match. 149*0Sstevel@tonic-gate */ 150*0Sstevel@tonic-gate self = curthread; 151*0Sstevel@tonic-gate if ((nlocks = self->ul_rdlocks) != 0) 152*0Sstevel@tonic-gate readlockp = self->ul_readlock.array; 153*0Sstevel@tonic-gate else { 154*0Sstevel@tonic-gate nlocks = 1; 155*0Sstevel@tonic-gate readlockp = &self->ul_readlock.single; 156*0Sstevel@tonic-gate } 157*0Sstevel@tonic-gate 158*0Sstevel@tonic-gate for (; nlocks; nlocks--, readlockp++) 159*0Sstevel@tonic-gate if (readlockp->rd_rwlock == rwlp) 160*0Sstevel@tonic-gate return (readlockp->rd_count? 1 : 0); 161*0Sstevel@tonic-gate 162*0Sstevel@tonic-gate return (0); 163*0Sstevel@tonic-gate } 164*0Sstevel@tonic-gate 165*0Sstevel@tonic-gate /* 166*0Sstevel@tonic-gate * Check if a writer version of the lock is held by the current thread. 167*0Sstevel@tonic-gate * rw_write_is_held() is private to libc. 168*0Sstevel@tonic-gate */ 169*0Sstevel@tonic-gate #pragma weak rw_write_is_held = _rw_write_held 170*0Sstevel@tonic-gate #pragma weak rw_write_held = _rw_write_held 171*0Sstevel@tonic-gate int 172*0Sstevel@tonic-gate _rw_write_held(rwlock_t *rwlp) 173*0Sstevel@tonic-gate { 174*0Sstevel@tonic-gate ulwp_t *self = curthread; 175*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 176*0Sstevel@tonic-gate 177*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) 178*0Sstevel@tonic-gate return (((uint32_t)rwlp->rwlock_readers & URW_WRITE_LOCKED) && 179*0Sstevel@tonic-gate (rwlp->rwlock_ownerpid == udp->pid) && 180*0Sstevel@tonic-gate (rwlp->rwlock_owner == (uintptr_t)self)); 181*0Sstevel@tonic-gate 182*0Sstevel@tonic-gate /* USYNC_THREAD */ 183*0Sstevel@tonic-gate return (rwlp->rwlock_readers == -1 && mutex_is_held(&rwlp->mutex)); 184*0Sstevel@tonic-gate } 185*0Sstevel@tonic-gate 186*0Sstevel@tonic-gate #pragma weak rwlock_init = __rwlock_init 187*0Sstevel@tonic-gate #pragma weak _rwlock_init = __rwlock_init 188*0Sstevel@tonic-gate /* ARGSUSED2 */ 189*0Sstevel@tonic-gate int 190*0Sstevel@tonic-gate __rwlock_init(rwlock_t *rwlp, int type, void *arg) 191*0Sstevel@tonic-gate { 192*0Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 193*0Sstevel@tonic-gate return (EINVAL); 194*0Sstevel@tonic-gate /* 195*0Sstevel@tonic-gate * Once reinitialized, we can no longer be holding a read or write lock. 196*0Sstevel@tonic-gate * We can do nothing about other threads that are holding read locks. 197*0Sstevel@tonic-gate */ 198*0Sstevel@tonic-gate if (rw_read_is_held(rwlp)) 199*0Sstevel@tonic-gate rwl_entry(rwlp)->rd_count = 0; 200*0Sstevel@tonic-gate (void) _memset(rwlp, 0, sizeof (*rwlp)); 201*0Sstevel@tonic-gate rwlp->rwlock_type = (uint16_t)type; 202*0Sstevel@tonic-gate rwlp->rwlock_magic = RWL_MAGIC; 203*0Sstevel@tonic-gate rwlp->rwlock_readers = 0; 204*0Sstevel@tonic-gate rwlp->mutex.mutex_type = (uint8_t)type; 205*0Sstevel@tonic-gate rwlp->mutex.mutex_flag = LOCK_INITED; 206*0Sstevel@tonic-gate rwlp->mutex.mutex_magic = MUTEX_MAGIC; 207*0Sstevel@tonic-gate rwlp->readercv.cond_type = (uint16_t)type; 208*0Sstevel@tonic-gate rwlp->readercv.cond_magic = COND_MAGIC; 209*0Sstevel@tonic-gate rwlp->writercv.cond_type = (uint16_t)type; 210*0Sstevel@tonic-gate rwlp->writercv.cond_magic = COND_MAGIC; 211*0Sstevel@tonic-gate return (0); 212*0Sstevel@tonic-gate } 213*0Sstevel@tonic-gate 214*0Sstevel@tonic-gate #pragma weak rwlock_destroy = __rwlock_destroy 215*0Sstevel@tonic-gate #pragma weak _rwlock_destroy = __rwlock_destroy 216*0Sstevel@tonic-gate #pragma weak pthread_rwlock_destroy = __rwlock_destroy 217*0Sstevel@tonic-gate #pragma weak _pthread_rwlock_destroy = __rwlock_destroy 218*0Sstevel@tonic-gate int 219*0Sstevel@tonic-gate __rwlock_destroy(rwlock_t *rwlp) 220*0Sstevel@tonic-gate { 221*0Sstevel@tonic-gate /* 222*0Sstevel@tonic-gate * Once destroyed, we can no longer be holding a read or write lock. 223*0Sstevel@tonic-gate * We can do nothing about other threads that are holding read locks. 224*0Sstevel@tonic-gate */ 225*0Sstevel@tonic-gate if (rw_read_is_held(rwlp)) 226*0Sstevel@tonic-gate rwl_entry(rwlp)->rd_count = 0; 227*0Sstevel@tonic-gate rwlp->rwlock_magic = 0; 228*0Sstevel@tonic-gate tdb_sync_obj_deregister(rwlp); 229*0Sstevel@tonic-gate return (0); 230*0Sstevel@tonic-gate } 231*0Sstevel@tonic-gate 232*0Sstevel@tonic-gate /* 233*0Sstevel@tonic-gate * Wake up the next thread sleeping on the rwlock queue and then 234*0Sstevel@tonic-gate * drop the queue lock. Return non-zero if we wake up someone. 235*0Sstevel@tonic-gate * 236*0Sstevel@tonic-gate * This is called whenever a thread releases the lock and whenever a 237*0Sstevel@tonic-gate * thread successfully or unsuccessfully attempts to acquire the lock. 238*0Sstevel@tonic-gate * (Basically, whenever the state of the queue might have changed.) 239*0Sstevel@tonic-gate * 240*0Sstevel@tonic-gate * We wake up at most one thread. If there are more threads to be 241*0Sstevel@tonic-gate * awakened, the next one will be waked up by the thread we wake up. 242*0Sstevel@tonic-gate * This ensures that queued threads will acquire the lock in priority 243*0Sstevel@tonic-gate * order and that queued writers will take precedence over queued 244*0Sstevel@tonic-gate * readers of the same priority. 245*0Sstevel@tonic-gate */ 246*0Sstevel@tonic-gate static int 247*0Sstevel@tonic-gate rw_queue_release(queue_head_t *qp, rwlock_t *rwlp) 248*0Sstevel@tonic-gate { 249*0Sstevel@tonic-gate ulwp_t *ulwp; 250*0Sstevel@tonic-gate int more; 251*0Sstevel@tonic-gate 252*0Sstevel@tonic-gate if (rwlp->rwlock_readers >= 0 && rwlp->rwlock_mwaiters) { 253*0Sstevel@tonic-gate /* 254*0Sstevel@tonic-gate * The lock is free or at least is available to readers 255*0Sstevel@tonic-gate * and there are (or might be) waiters on the queue. 256*0Sstevel@tonic-gate */ 257*0Sstevel@tonic-gate if (rwlp->rwlock_readers != 0 && 258*0Sstevel@tonic-gate (ulwp = queue_waiter(qp, rwlp)) == NULL) 259*0Sstevel@tonic-gate rwlp->rwlock_mwaiters = 0; 260*0Sstevel@tonic-gate else if (rwlp->rwlock_readers == 0 || !ulwp->ul_writer) { 261*0Sstevel@tonic-gate if ((ulwp = dequeue(qp, rwlp, &more)) == NULL) 262*0Sstevel@tonic-gate rwlp->rwlock_mwaiters = 0; 263*0Sstevel@tonic-gate else { 264*0Sstevel@tonic-gate ulwp_t *self = curthread; 265*0Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 266*0Sstevel@tonic-gate 267*0Sstevel@tonic-gate rwlp->rwlock_mwaiters = (more? 1 : 0); 268*0Sstevel@tonic-gate no_preempt(self); 269*0Sstevel@tonic-gate queue_unlock(qp); 270*0Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 271*0Sstevel@tonic-gate preempt(self); 272*0Sstevel@tonic-gate return (1); 273*0Sstevel@tonic-gate } 274*0Sstevel@tonic-gate } 275*0Sstevel@tonic-gate } 276*0Sstevel@tonic-gate queue_unlock(qp); 277*0Sstevel@tonic-gate return (0); 278*0Sstevel@tonic-gate } 279*0Sstevel@tonic-gate 280*0Sstevel@tonic-gate /* 281*0Sstevel@tonic-gate * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock, 282*0Sstevel@tonic-gate * and trywrlock for process-shared (USYNC_PROCESS) rwlocks. 283*0Sstevel@tonic-gate * 284*0Sstevel@tonic-gate * Note: if the lock appears to be contended we call __lwp_rwlock_rdlock() 285*0Sstevel@tonic-gate * or __lwp_rwlock_wrlock() holding the mutex. These return with the mutex 286*0Sstevel@tonic-gate * released, and if they need to sleep will release the mutex first. In the 287*0Sstevel@tonic-gate * event of a spurious wakeup, these will return EAGAIN (because it is much 288*0Sstevel@tonic-gate * easier for us to re-acquire the mutex here). 289*0Sstevel@tonic-gate */ 290*0Sstevel@tonic-gate int 291*0Sstevel@tonic-gate shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr) 292*0Sstevel@tonic-gate { 293*0Sstevel@tonic-gate uint32_t *rwstate = (uint32_t *)&rwlp->readers; 294*0Sstevel@tonic-gate ulwp_t *self = curthread; 295*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 296*0Sstevel@tonic-gate int try_flag; 297*0Sstevel@tonic-gate int error = 0; 298*0Sstevel@tonic-gate 299*0Sstevel@tonic-gate try_flag = (rd_wr & TRY_FLAG); 300*0Sstevel@tonic-gate rd_wr &= ~TRY_FLAG; 301*0Sstevel@tonic-gate ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK); 302*0Sstevel@tonic-gate 303*0Sstevel@tonic-gate if (!try_flag) { 304*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr); 305*0Sstevel@tonic-gate } 306*0Sstevel@tonic-gate 307*0Sstevel@tonic-gate do { 308*0Sstevel@tonic-gate if ((error = _private_mutex_lock(&rwlp->mutex)) != 0) 309*0Sstevel@tonic-gate break; 310*0Sstevel@tonic-gate 311*0Sstevel@tonic-gate if (rd_wr == READ_LOCK) { 312*0Sstevel@tonic-gate /* 313*0Sstevel@tonic-gate * We are a reader. 314*0Sstevel@tonic-gate */ 315*0Sstevel@tonic-gate 316*0Sstevel@tonic-gate if ((*rwstate & ~URW_READERS_MASK) == 0) { 317*0Sstevel@tonic-gate (*rwstate)++; 318*0Sstevel@tonic-gate (void) _private_mutex_unlock(&rwlp->mutex); 319*0Sstevel@tonic-gate } else if (try_flag) { 320*0Sstevel@tonic-gate if (*rwstate & URW_WRITE_LOCKED) { 321*0Sstevel@tonic-gate error = EBUSY; 322*0Sstevel@tonic-gate (void) _private_mutex_unlock( 323*0Sstevel@tonic-gate &rwlp->mutex); 324*0Sstevel@tonic-gate } else { 325*0Sstevel@tonic-gate /* 326*0Sstevel@tonic-gate * We have a higher priority than any 327*0Sstevel@tonic-gate * queued waiters, or the waiters bit 328*0Sstevel@tonic-gate * may be inaccurate. Only the kernel 329*0Sstevel@tonic-gate * knows for sure. 330*0Sstevel@tonic-gate */ 331*0Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 332*0Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 333*0Sstevel@tonic-gate error = __lwp_rwlock_tryrdlock(rwlp); 334*0Sstevel@tonic-gate } 335*0Sstevel@tonic-gate } else { 336*0Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 337*0Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 338*0Sstevel@tonic-gate error = __lwp_rwlock_rdlock(rwlp, tsp); 339*0Sstevel@tonic-gate } 340*0Sstevel@tonic-gate } else { 341*0Sstevel@tonic-gate /* 342*0Sstevel@tonic-gate * We are a writer. 343*0Sstevel@tonic-gate */ 344*0Sstevel@tonic-gate 345*0Sstevel@tonic-gate if (*rwstate == 0) { 346*0Sstevel@tonic-gate *rwstate = URW_WRITE_LOCKED; 347*0Sstevel@tonic-gate (void) _private_mutex_unlock(&rwlp->mutex); 348*0Sstevel@tonic-gate } else if (try_flag) { 349*0Sstevel@tonic-gate if (*rwstate & URW_WRITE_LOCKED) { 350*0Sstevel@tonic-gate error = EBUSY; 351*0Sstevel@tonic-gate (void) _private_mutex_unlock( 352*0Sstevel@tonic-gate &rwlp->mutex); 353*0Sstevel@tonic-gate } else { 354*0Sstevel@tonic-gate /* 355*0Sstevel@tonic-gate * The waiters bit may be inaccurate. 356*0Sstevel@tonic-gate * Only the kernel knows for sure. 357*0Sstevel@tonic-gate */ 358*0Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 359*0Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 360*0Sstevel@tonic-gate error = __lwp_rwlock_trywrlock(rwlp); 361*0Sstevel@tonic-gate } 362*0Sstevel@tonic-gate } else { 363*0Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 364*0Sstevel@tonic-gate rwlp->rwlock_mownerpid = 0; 365*0Sstevel@tonic-gate error = __lwp_rwlock_wrlock(rwlp, tsp); 366*0Sstevel@tonic-gate } 367*0Sstevel@tonic-gate } 368*0Sstevel@tonic-gate } while (error == EAGAIN); 369*0Sstevel@tonic-gate 370*0Sstevel@tonic-gate if (error == 0) { 371*0Sstevel@tonic-gate if (rd_wr == WRITE_LOCK) { 372*0Sstevel@tonic-gate rwlp->rwlock_owner = (uintptr_t)self; 373*0Sstevel@tonic-gate rwlp->rwlock_ownerpid = udp->pid; 374*0Sstevel@tonic-gate } 375*0Sstevel@tonic-gate if (!try_flag) { 376*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 1); 377*0Sstevel@tonic-gate } 378*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr); 379*0Sstevel@tonic-gate } else if (!try_flag) { 380*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 0); 381*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__error, rwlp, rd_wr, error); 382*0Sstevel@tonic-gate } 383*0Sstevel@tonic-gate return (error); 384*0Sstevel@tonic-gate } 385*0Sstevel@tonic-gate 386*0Sstevel@tonic-gate /* 387*0Sstevel@tonic-gate * Code for unlock of process-shared (USYNC_PROCESS) rwlocks. 388*0Sstevel@tonic-gate * 389*0Sstevel@tonic-gate * Note: if the lock appears to have waiters we call __lwp_rwlock_unlock() 390*0Sstevel@tonic-gate * holding the mutex. This returns with the mutex still held (for us to 391*0Sstevel@tonic-gate * release). 392*0Sstevel@tonic-gate */ 393*0Sstevel@tonic-gate int 394*0Sstevel@tonic-gate shared_rwlock_unlock(rwlock_t *rwlp, int *waked) 395*0Sstevel@tonic-gate { 396*0Sstevel@tonic-gate uint32_t *rwstate = (uint32_t *)&rwlp->readers; 397*0Sstevel@tonic-gate int error = 0; 398*0Sstevel@tonic-gate 399*0Sstevel@tonic-gate if ((error = _private_mutex_lock(&rwlp->mutex)) != 0) 400*0Sstevel@tonic-gate return (error); 401*0Sstevel@tonic-gate 402*0Sstevel@tonic-gate /* Reset flag used to suggest caller yields. */ 403*0Sstevel@tonic-gate *waked = 0; 404*0Sstevel@tonic-gate 405*0Sstevel@tonic-gate /* Our right to unlock was checked in __rw_unlock(). */ 406*0Sstevel@tonic-gate if (*rwstate & URW_WRITE_LOCKED) { 407*0Sstevel@tonic-gate rwlp->rwlock_owner = 0; 408*0Sstevel@tonic-gate rwlp->rwlock_ownerpid = 0; 409*0Sstevel@tonic-gate } 410*0Sstevel@tonic-gate 411*0Sstevel@tonic-gate if ((*rwstate & ~URW_READERS_MASK) == 0) { 412*0Sstevel@tonic-gate /* Simple multiple readers, no waiters case. */ 413*0Sstevel@tonic-gate if (*rwstate > 0) 414*0Sstevel@tonic-gate (*rwstate)--; 415*0Sstevel@tonic-gate } else if (!(*rwstate & URW_HAS_WAITERS)) { 416*0Sstevel@tonic-gate /* Simple no waiters case (i.e. was write locked). */ 417*0Sstevel@tonic-gate *rwstate = 0; 418*0Sstevel@tonic-gate } else { 419*0Sstevel@tonic-gate /* 420*0Sstevel@tonic-gate * We appear to have waiters so we must call into the kernel. 421*0Sstevel@tonic-gate * If there are waiters a full handoff will occur (rwstate 422*0Sstevel@tonic-gate * will be updated, and one or more threads will be awoken). 423*0Sstevel@tonic-gate */ 424*0Sstevel@tonic-gate error = __lwp_rwlock_unlock(rwlp); 425*0Sstevel@tonic-gate 426*0Sstevel@tonic-gate /* Suggest caller yields. */ 427*0Sstevel@tonic-gate *waked = 1; 428*0Sstevel@tonic-gate } 429*0Sstevel@tonic-gate 430*0Sstevel@tonic-gate (void) _private_mutex_unlock(&rwlp->mutex); 431*0Sstevel@tonic-gate 432*0Sstevel@tonic-gate if (error) { 433*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__error, rwlp, 0, error); 434*0Sstevel@tonic-gate } else { 435*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 436*0Sstevel@tonic-gate } 437*0Sstevel@tonic-gate 438*0Sstevel@tonic-gate return (error); 439*0Sstevel@tonic-gate } 440*0Sstevel@tonic-gate 441*0Sstevel@tonic-gate 442*0Sstevel@tonic-gate /* 443*0Sstevel@tonic-gate * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock, 444*0Sstevel@tonic-gate * and trywrlock for process-private (USYNC_THREAD) rwlocks. 445*0Sstevel@tonic-gate */ 446*0Sstevel@tonic-gate int 447*0Sstevel@tonic-gate rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr) 448*0Sstevel@tonic-gate { 449*0Sstevel@tonic-gate ulwp_t *self = curthread; 450*0Sstevel@tonic-gate queue_head_t *qp; 451*0Sstevel@tonic-gate ulwp_t *ulwp; 452*0Sstevel@tonic-gate int try_flag; 453*0Sstevel@tonic-gate int error = 0; 454*0Sstevel@tonic-gate 455*0Sstevel@tonic-gate try_flag = (rd_wr & TRY_FLAG); 456*0Sstevel@tonic-gate rd_wr &= ~TRY_FLAG; 457*0Sstevel@tonic-gate ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK); 458*0Sstevel@tonic-gate 459*0Sstevel@tonic-gate /* 460*0Sstevel@tonic-gate * Optimize for the case of having only a single thread. 461*0Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 462*0Sstevel@tonic-gate * We don't need the protection of queue_lock() in this case. 463*0Sstevel@tonic-gate * We need to defer signals, however (the other form of concurrency). 464*0Sstevel@tonic-gate */ 465*0Sstevel@tonic-gate if (!self->ul_uberdata->uberflags.uf_mt) { 466*0Sstevel@tonic-gate sigoff(self); 467*0Sstevel@tonic-gate if (rwlp->rwlock_readers < 0 || 468*0Sstevel@tonic-gate (rd_wr == WRITE_LOCK && rwlp->rwlock_readers != 0)) { 469*0Sstevel@tonic-gate sigon(self); 470*0Sstevel@tonic-gate if (try_flag) 471*0Sstevel@tonic-gate return (EBUSY); 472*0Sstevel@tonic-gate /* 473*0Sstevel@tonic-gate * Sombody other than ourself owns the lock. (If we 474*0Sstevel@tonic-gate * owned the lock, either for reading or writing, we 475*0Sstevel@tonic-gate * would already have returned EDEADLK in our caller.) 476*0Sstevel@tonic-gate * This can happen only in the child of fork1() when 477*0Sstevel@tonic-gate * some now-defunct thread was holding the lock when 478*0Sstevel@tonic-gate * the fork1() was executed by the current thread. 479*0Sstevel@tonic-gate * In this case, we just fall into the long way 480*0Sstevel@tonic-gate * to block, either forever or with a timeout. 481*0Sstevel@tonic-gate */ 482*0Sstevel@tonic-gate ASSERT(MUTEX_OWNER(&rwlp->mutex) != self); 483*0Sstevel@tonic-gate } else { 484*0Sstevel@tonic-gate if (rd_wr == READ_LOCK) 485*0Sstevel@tonic-gate rwlp->rwlock_readers++; 486*0Sstevel@tonic-gate else { 487*0Sstevel@tonic-gate rwlp->rwlock_readers = -1; 488*0Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKSET; 489*0Sstevel@tonic-gate rwlp->rwlock_mowner = (uintptr_t)self; 490*0Sstevel@tonic-gate } 491*0Sstevel@tonic-gate sigon(self); 492*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr); 493*0Sstevel@tonic-gate return (0); 494*0Sstevel@tonic-gate } 495*0Sstevel@tonic-gate } 496*0Sstevel@tonic-gate 497*0Sstevel@tonic-gate if (!try_flag) { 498*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr); 499*0Sstevel@tonic-gate } 500*0Sstevel@tonic-gate 501*0Sstevel@tonic-gate /* 502*0Sstevel@tonic-gate * Do it the long way. 503*0Sstevel@tonic-gate */ 504*0Sstevel@tonic-gate qp = queue_lock(rwlp, MX); 505*0Sstevel@tonic-gate while (error == 0) { 506*0Sstevel@tonic-gate if (rwlp->rwlock_readers < 0 || 507*0Sstevel@tonic-gate (rd_wr == WRITE_LOCK && rwlp->rwlock_readers != 0)) 508*0Sstevel@tonic-gate /* EMPTY */; /* somebody holds the lock */ 509*0Sstevel@tonic-gate else if (!rwlp->rwlock_mwaiters) 510*0Sstevel@tonic-gate break; /* no queued waiters */ 511*0Sstevel@tonic-gate else if ((ulwp = queue_waiter(qp, rwlp)) == NULL) { 512*0Sstevel@tonic-gate rwlp->rwlock_mwaiters = 0; 513*0Sstevel@tonic-gate break; /* no queued waiters */ 514*0Sstevel@tonic-gate } else { 515*0Sstevel@tonic-gate int our_pri = real_priority(self); 516*0Sstevel@tonic-gate int his_pri = real_priority(ulwp); 517*0Sstevel@tonic-gate 518*0Sstevel@tonic-gate if (rd_wr == WRITE_LOCK) { 519*0Sstevel@tonic-gate /* 520*0Sstevel@tonic-gate * We defer to a queued thread that has 521*0Sstevel@tonic-gate * a higher priority than ours. 522*0Sstevel@tonic-gate */ 523*0Sstevel@tonic-gate if (his_pri <= our_pri) 524*0Sstevel@tonic-gate break; 525*0Sstevel@tonic-gate } else { 526*0Sstevel@tonic-gate /* 527*0Sstevel@tonic-gate * We defer to a queued thread that has 528*0Sstevel@tonic-gate * a higher priority than ours or that 529*0Sstevel@tonic-gate * is a writer whose priority equals ours. 530*0Sstevel@tonic-gate */ 531*0Sstevel@tonic-gate if (his_pri < our_pri || 532*0Sstevel@tonic-gate (his_pri == our_pri && !ulwp->ul_writer)) 533*0Sstevel@tonic-gate break; 534*0Sstevel@tonic-gate } 535*0Sstevel@tonic-gate } 536*0Sstevel@tonic-gate /* 537*0Sstevel@tonic-gate * We are about to block. 538*0Sstevel@tonic-gate * If we're doing a trylock, return EBUSY instead. 539*0Sstevel@tonic-gate */ 540*0Sstevel@tonic-gate if (try_flag) { 541*0Sstevel@tonic-gate error = EBUSY; 542*0Sstevel@tonic-gate break; 543*0Sstevel@tonic-gate } 544*0Sstevel@tonic-gate /* 545*0Sstevel@tonic-gate * Enqueue writers ahead of readers of the 546*0Sstevel@tonic-gate * same priority. 547*0Sstevel@tonic-gate */ 548*0Sstevel@tonic-gate self->ul_writer = rd_wr; /* *must* be 0 or 1 */ 549*0Sstevel@tonic-gate enqueue(qp, self, rwlp, MX); 550*0Sstevel@tonic-gate rwlp->rwlock_mwaiters = 1; 551*0Sstevel@tonic-gate set_parking_flag(self, 1); 552*0Sstevel@tonic-gate queue_unlock(qp); 553*0Sstevel@tonic-gate if ((error = __lwp_park(tsp, 0)) == EINTR) 554*0Sstevel@tonic-gate error = 0; 555*0Sstevel@tonic-gate self->ul_writer = 0; 556*0Sstevel@tonic-gate set_parking_flag(self, 0); 557*0Sstevel@tonic-gate qp = queue_lock(rwlp, MX); 558*0Sstevel@tonic-gate if (self->ul_sleepq) /* timeout or spurious wakeup */ 559*0Sstevel@tonic-gate rwlp->rwlock_mwaiters = dequeue_self(qp, rwlp); 560*0Sstevel@tonic-gate } 561*0Sstevel@tonic-gate 562*0Sstevel@tonic-gate if (error == 0) { 563*0Sstevel@tonic-gate if (rd_wr == READ_LOCK) 564*0Sstevel@tonic-gate rwlp->rwlock_readers++; 565*0Sstevel@tonic-gate else { 566*0Sstevel@tonic-gate rwlp->rwlock_readers = -1; 567*0Sstevel@tonic-gate /* make it look like we acquired the embedded mutex */ 568*0Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKSET; 569*0Sstevel@tonic-gate rwlp->rwlock_mowner = (uintptr_t)self; 570*0Sstevel@tonic-gate } 571*0Sstevel@tonic-gate if (!try_flag) { 572*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 1); 573*0Sstevel@tonic-gate } 574*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr); 575*0Sstevel@tonic-gate } else if (!try_flag) { 576*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 0); 577*0Sstevel@tonic-gate DTRACE_PROBE3(plockstat, rw__error, rwlp, rd_wr, error); 578*0Sstevel@tonic-gate } 579*0Sstevel@tonic-gate 580*0Sstevel@tonic-gate (void) rw_queue_release(qp, rwlp); 581*0Sstevel@tonic-gate 582*0Sstevel@tonic-gate return (error); 583*0Sstevel@tonic-gate } 584*0Sstevel@tonic-gate 585*0Sstevel@tonic-gate int 586*0Sstevel@tonic-gate rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp) 587*0Sstevel@tonic-gate { 588*0Sstevel@tonic-gate ulwp_t *self = curthread; 589*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 590*0Sstevel@tonic-gate readlock_t *readlockp; 591*0Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 592*0Sstevel@tonic-gate int error; 593*0Sstevel@tonic-gate 594*0Sstevel@tonic-gate /* 595*0Sstevel@tonic-gate * If we already hold a readers lock on this rwlock, 596*0Sstevel@tonic-gate * just increment our reference count and return. 597*0Sstevel@tonic-gate */ 598*0Sstevel@tonic-gate readlockp = rwl_entry(rwlp); 599*0Sstevel@tonic-gate if (readlockp->rd_count != 0) { 600*0Sstevel@tonic-gate if (readlockp->rd_count == READ_LOCK_MAX) 601*0Sstevel@tonic-gate return (EAGAIN); 602*0Sstevel@tonic-gate readlockp->rd_count++; 603*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK); 604*0Sstevel@tonic-gate return (0); 605*0Sstevel@tonic-gate } 606*0Sstevel@tonic-gate 607*0Sstevel@tonic-gate /* 608*0Sstevel@tonic-gate * If we hold the writer lock, bail out. 609*0Sstevel@tonic-gate */ 610*0Sstevel@tonic-gate if (rw_write_is_held(rwlp)) { 611*0Sstevel@tonic-gate if (self->ul_error_detection) 612*0Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_rdlock", 613*0Sstevel@tonic-gate "calling thread owns the writer lock"); 614*0Sstevel@tonic-gate return (EDEADLK); 615*0Sstevel@tonic-gate } 616*0Sstevel@tonic-gate 617*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */ 618*0Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, tsp, READ_LOCK); 619*0Sstevel@tonic-gate else /* user-level */ 620*0Sstevel@tonic-gate error = rwlock_lock(rwlp, tsp, READ_LOCK); 621*0Sstevel@tonic-gate 622*0Sstevel@tonic-gate if (error == 0) { 623*0Sstevel@tonic-gate readlockp->rd_count = 1; 624*0Sstevel@tonic-gate if (rwsp) 625*0Sstevel@tonic-gate tdb_incr(rwsp->rw_rdlock); 626*0Sstevel@tonic-gate } 627*0Sstevel@tonic-gate 628*0Sstevel@tonic-gate return (error); 629*0Sstevel@tonic-gate } 630*0Sstevel@tonic-gate 631*0Sstevel@tonic-gate #pragma weak rw_rdlock = __rw_rdlock 632*0Sstevel@tonic-gate #pragma weak _rw_rdlock = __rw_rdlock 633*0Sstevel@tonic-gate #pragma weak pthread_rwlock_rdlock = __rw_rdlock 634*0Sstevel@tonic-gate #pragma weak _pthread_rwlock_rdlock = __rw_rdlock 635*0Sstevel@tonic-gate int 636*0Sstevel@tonic-gate __rw_rdlock(rwlock_t *rwlp) 637*0Sstevel@tonic-gate { 638*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 639*0Sstevel@tonic-gate return (rw_rdlock_impl(rwlp, NULL)); 640*0Sstevel@tonic-gate } 641*0Sstevel@tonic-gate 642*0Sstevel@tonic-gate void 643*0Sstevel@tonic-gate lrw_rdlock(rwlock_t *rwlp) 644*0Sstevel@tonic-gate { 645*0Sstevel@tonic-gate enter_critical(curthread); 646*0Sstevel@tonic-gate (void) rw_rdlock_impl(rwlp, NULL); 647*0Sstevel@tonic-gate } 648*0Sstevel@tonic-gate 649*0Sstevel@tonic-gate #pragma weak pthread_rwlock_reltimedrdlock_np = \ 650*0Sstevel@tonic-gate _pthread_rwlock_reltimedrdlock_np 651*0Sstevel@tonic-gate int 652*0Sstevel@tonic-gate _pthread_rwlock_reltimedrdlock_np(rwlock_t *rwlp, const timespec_t *reltime) 653*0Sstevel@tonic-gate { 654*0Sstevel@tonic-gate timespec_t tslocal = *reltime; 655*0Sstevel@tonic-gate int error; 656*0Sstevel@tonic-gate 657*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 658*0Sstevel@tonic-gate error = rw_rdlock_impl(rwlp, &tslocal); 659*0Sstevel@tonic-gate if (error == ETIME) 660*0Sstevel@tonic-gate error = ETIMEDOUT; 661*0Sstevel@tonic-gate return (error); 662*0Sstevel@tonic-gate } 663*0Sstevel@tonic-gate 664*0Sstevel@tonic-gate #pragma weak pthread_rwlock_timedrdlock = _pthread_rwlock_timedrdlock 665*0Sstevel@tonic-gate int 666*0Sstevel@tonic-gate _pthread_rwlock_timedrdlock(rwlock_t *rwlp, const timespec_t *abstime) 667*0Sstevel@tonic-gate { 668*0Sstevel@tonic-gate timespec_t tslocal; 669*0Sstevel@tonic-gate int error; 670*0Sstevel@tonic-gate 671*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 672*0Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 673*0Sstevel@tonic-gate error = rw_rdlock_impl(rwlp, &tslocal); 674*0Sstevel@tonic-gate if (error == ETIME) 675*0Sstevel@tonic-gate error = ETIMEDOUT; 676*0Sstevel@tonic-gate return (error); 677*0Sstevel@tonic-gate } 678*0Sstevel@tonic-gate 679*0Sstevel@tonic-gate int 680*0Sstevel@tonic-gate rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp) 681*0Sstevel@tonic-gate { 682*0Sstevel@tonic-gate ulwp_t *self = curthread; 683*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 684*0Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 685*0Sstevel@tonic-gate int error; 686*0Sstevel@tonic-gate 687*0Sstevel@tonic-gate /* 688*0Sstevel@tonic-gate * If we hold a readers lock on this rwlock, bail out. 689*0Sstevel@tonic-gate */ 690*0Sstevel@tonic-gate if (rw_read_is_held(rwlp)) { 691*0Sstevel@tonic-gate if (self->ul_error_detection) 692*0Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_wrlock", 693*0Sstevel@tonic-gate "calling thread owns the readers lock"); 694*0Sstevel@tonic-gate return (EDEADLK); 695*0Sstevel@tonic-gate } 696*0Sstevel@tonic-gate 697*0Sstevel@tonic-gate /* 698*0Sstevel@tonic-gate * If we hold the writer lock, bail out. 699*0Sstevel@tonic-gate */ 700*0Sstevel@tonic-gate if (rw_write_is_held(rwlp)) { 701*0Sstevel@tonic-gate if (self->ul_error_detection) 702*0Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_wrlock", 703*0Sstevel@tonic-gate "calling thread owns the writer lock"); 704*0Sstevel@tonic-gate return (EDEADLK); 705*0Sstevel@tonic-gate } 706*0Sstevel@tonic-gate 707*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */ 708*0Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, tsp, WRITE_LOCK); 709*0Sstevel@tonic-gate } else { /* user-level */ 710*0Sstevel@tonic-gate error = rwlock_lock(rwlp, tsp, WRITE_LOCK); 711*0Sstevel@tonic-gate } 712*0Sstevel@tonic-gate 713*0Sstevel@tonic-gate if (error == 0 && rwsp) { 714*0Sstevel@tonic-gate tdb_incr(rwsp->rw_wrlock); 715*0Sstevel@tonic-gate rwsp->rw_wrlock_begin_hold = gethrtime(); 716*0Sstevel@tonic-gate } 717*0Sstevel@tonic-gate 718*0Sstevel@tonic-gate return (error); 719*0Sstevel@tonic-gate } 720*0Sstevel@tonic-gate 721*0Sstevel@tonic-gate #pragma weak rw_wrlock = __rw_wrlock 722*0Sstevel@tonic-gate #pragma weak _rw_wrlock = __rw_wrlock 723*0Sstevel@tonic-gate #pragma weak pthread_rwlock_wrlock = __rw_wrlock 724*0Sstevel@tonic-gate #pragma weak _pthread_rwlock_wrlock = __rw_wrlock 725*0Sstevel@tonic-gate int 726*0Sstevel@tonic-gate __rw_wrlock(rwlock_t *rwlp) 727*0Sstevel@tonic-gate { 728*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 729*0Sstevel@tonic-gate return (rw_wrlock_impl(rwlp, NULL)); 730*0Sstevel@tonic-gate } 731*0Sstevel@tonic-gate 732*0Sstevel@tonic-gate void 733*0Sstevel@tonic-gate lrw_wrlock(rwlock_t *rwlp) 734*0Sstevel@tonic-gate { 735*0Sstevel@tonic-gate enter_critical(curthread); 736*0Sstevel@tonic-gate (void) rw_wrlock_impl(rwlp, NULL); 737*0Sstevel@tonic-gate } 738*0Sstevel@tonic-gate 739*0Sstevel@tonic-gate #pragma weak pthread_rwlock_reltimedwrlock_np = \ 740*0Sstevel@tonic-gate _pthread_rwlock_reltimedwrlock_np 741*0Sstevel@tonic-gate int 742*0Sstevel@tonic-gate _pthread_rwlock_reltimedwrlock_np(rwlock_t *rwlp, const timespec_t *reltime) 743*0Sstevel@tonic-gate { 744*0Sstevel@tonic-gate timespec_t tslocal = *reltime; 745*0Sstevel@tonic-gate int error; 746*0Sstevel@tonic-gate 747*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 748*0Sstevel@tonic-gate error = rw_wrlock_impl(rwlp, &tslocal); 749*0Sstevel@tonic-gate if (error == ETIME) 750*0Sstevel@tonic-gate error = ETIMEDOUT; 751*0Sstevel@tonic-gate return (error); 752*0Sstevel@tonic-gate } 753*0Sstevel@tonic-gate 754*0Sstevel@tonic-gate #pragma weak pthread_rwlock_timedwrlock = _pthread_rwlock_timedwrlock 755*0Sstevel@tonic-gate int 756*0Sstevel@tonic-gate _pthread_rwlock_timedwrlock(rwlock_t *rwlp, const timespec_t *abstime) 757*0Sstevel@tonic-gate { 758*0Sstevel@tonic-gate timespec_t tslocal; 759*0Sstevel@tonic-gate int error; 760*0Sstevel@tonic-gate 761*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 762*0Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 763*0Sstevel@tonic-gate error = rw_wrlock_impl(rwlp, &tslocal); 764*0Sstevel@tonic-gate if (error == ETIME) 765*0Sstevel@tonic-gate error = ETIMEDOUT; 766*0Sstevel@tonic-gate return (error); 767*0Sstevel@tonic-gate } 768*0Sstevel@tonic-gate 769*0Sstevel@tonic-gate #pragma weak rw_tryrdlock = __rw_tryrdlock 770*0Sstevel@tonic-gate #pragma weak _rw_tryrdlock = __rw_tryrdlock 771*0Sstevel@tonic-gate #pragma weak pthread_rwlock_tryrdlock = __rw_tryrdlock 772*0Sstevel@tonic-gate #pragma weak _pthread_rwlock_tryrdlock = __rw_tryrdlock 773*0Sstevel@tonic-gate int 774*0Sstevel@tonic-gate __rw_tryrdlock(rwlock_t *rwlp) 775*0Sstevel@tonic-gate { 776*0Sstevel@tonic-gate ulwp_t *self = curthread; 777*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 778*0Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 779*0Sstevel@tonic-gate readlock_t *readlockp; 780*0Sstevel@tonic-gate int error; 781*0Sstevel@tonic-gate 782*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 783*0Sstevel@tonic-gate 784*0Sstevel@tonic-gate if (rwsp) 785*0Sstevel@tonic-gate tdb_incr(rwsp->rw_rdlock_try); 786*0Sstevel@tonic-gate 787*0Sstevel@tonic-gate /* 788*0Sstevel@tonic-gate * If we already hold a readers lock on this rwlock, 789*0Sstevel@tonic-gate * just increment our reference count and return. 790*0Sstevel@tonic-gate */ 791*0Sstevel@tonic-gate readlockp = rwl_entry(rwlp); 792*0Sstevel@tonic-gate if (readlockp->rd_count != 0) { 793*0Sstevel@tonic-gate if (readlockp->rd_count == READ_LOCK_MAX) 794*0Sstevel@tonic-gate return (EAGAIN); 795*0Sstevel@tonic-gate readlockp->rd_count++; 796*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK); 797*0Sstevel@tonic-gate return (0); 798*0Sstevel@tonic-gate } 799*0Sstevel@tonic-gate 800*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */ 801*0Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, NULL, READ_LOCK_TRY); 802*0Sstevel@tonic-gate else /* user-level */ 803*0Sstevel@tonic-gate error = rwlock_lock(rwlp, NULL, READ_LOCK_TRY); 804*0Sstevel@tonic-gate 805*0Sstevel@tonic-gate if (error == 0) 806*0Sstevel@tonic-gate readlockp->rd_count = 1; 807*0Sstevel@tonic-gate else if (rwsp) 808*0Sstevel@tonic-gate tdb_incr(rwsp->rw_rdlock_try_fail); 809*0Sstevel@tonic-gate 810*0Sstevel@tonic-gate return (error); 811*0Sstevel@tonic-gate } 812*0Sstevel@tonic-gate 813*0Sstevel@tonic-gate #pragma weak rw_trywrlock = __rw_trywrlock 814*0Sstevel@tonic-gate #pragma weak _rw_trywrlock = __rw_trywrlock 815*0Sstevel@tonic-gate #pragma weak pthread_rwlock_trywrlock = __rw_trywrlock 816*0Sstevel@tonic-gate #pragma weak _pthread_rwlock_trywrlock = __rw_trywrlock 817*0Sstevel@tonic-gate int 818*0Sstevel@tonic-gate __rw_trywrlock(rwlock_t *rwlp) 819*0Sstevel@tonic-gate { 820*0Sstevel@tonic-gate ulwp_t *self = curthread; 821*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 822*0Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp); 823*0Sstevel@tonic-gate int error; 824*0Sstevel@tonic-gate 825*0Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 826*0Sstevel@tonic-gate 827*0Sstevel@tonic-gate if (rwsp) 828*0Sstevel@tonic-gate tdb_incr(rwsp->rw_wrlock_try); 829*0Sstevel@tonic-gate 830*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */ 831*0Sstevel@tonic-gate error = shared_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY); 832*0Sstevel@tonic-gate } else { /* user-level */ 833*0Sstevel@tonic-gate error = rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY); 834*0Sstevel@tonic-gate } 835*0Sstevel@tonic-gate if (rwsp) { 836*0Sstevel@tonic-gate if (error) 837*0Sstevel@tonic-gate tdb_incr(rwsp->rw_wrlock_try_fail); 838*0Sstevel@tonic-gate else 839*0Sstevel@tonic-gate rwsp->rw_wrlock_begin_hold = gethrtime(); 840*0Sstevel@tonic-gate } 841*0Sstevel@tonic-gate return (error); 842*0Sstevel@tonic-gate } 843*0Sstevel@tonic-gate 844*0Sstevel@tonic-gate #pragma weak rw_unlock = __rw_unlock 845*0Sstevel@tonic-gate #pragma weak _rw_unlock = __rw_unlock 846*0Sstevel@tonic-gate #pragma weak pthread_rwlock_unlock = __rw_unlock 847*0Sstevel@tonic-gate #pragma weak _pthread_rwlock_unlock = __rw_unlock 848*0Sstevel@tonic-gate int 849*0Sstevel@tonic-gate __rw_unlock(rwlock_t *rwlp) 850*0Sstevel@tonic-gate { 851*0Sstevel@tonic-gate ulwp_t *self = curthread; 852*0Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 853*0Sstevel@tonic-gate tdb_rwlock_stats_t *rwsp; 854*0Sstevel@tonic-gate int32_t lock_count; 855*0Sstevel@tonic-gate int waked; 856*0Sstevel@tonic-gate 857*0Sstevel@tonic-gate /* fetch the lock count once; it may change underfoot */ 858*0Sstevel@tonic-gate lock_count = rwlp->rwlock_readers; 859*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { 860*0Sstevel@tonic-gate /* munge it from rwstate */ 861*0Sstevel@tonic-gate if (lock_count & URW_WRITE_LOCKED) 862*0Sstevel@tonic-gate lock_count = -1; 863*0Sstevel@tonic-gate else 864*0Sstevel@tonic-gate lock_count &= URW_READERS_MASK; 865*0Sstevel@tonic-gate } 866*0Sstevel@tonic-gate 867*0Sstevel@tonic-gate if (lock_count < 0) { 868*0Sstevel@tonic-gate /* 869*0Sstevel@tonic-gate * Since the writer lock is held, we'd better be 870*0Sstevel@tonic-gate * holding it, else we cannot legitimately be here. 871*0Sstevel@tonic-gate */ 872*0Sstevel@tonic-gate if (!rw_write_is_held(rwlp)) { 873*0Sstevel@tonic-gate if (self->ul_error_detection) 874*0Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_unlock", 875*0Sstevel@tonic-gate "writer lock held, " 876*0Sstevel@tonic-gate "but not by the calling thread"); 877*0Sstevel@tonic-gate return (EPERM); 878*0Sstevel@tonic-gate } 879*0Sstevel@tonic-gate if ((rwsp = RWLOCK_STATS(rwlp, udp)) != NULL) { 880*0Sstevel@tonic-gate if (rwsp->rw_wrlock_begin_hold) 881*0Sstevel@tonic-gate rwsp->rw_wrlock_hold_time += 882*0Sstevel@tonic-gate gethrtime() - rwsp->rw_wrlock_begin_hold; 883*0Sstevel@tonic-gate rwsp->rw_wrlock_begin_hold = 0; 884*0Sstevel@tonic-gate } 885*0Sstevel@tonic-gate } else if (lock_count > 0) { 886*0Sstevel@tonic-gate /* 887*0Sstevel@tonic-gate * A readers lock is held; if we don't hold one, bail out. 888*0Sstevel@tonic-gate */ 889*0Sstevel@tonic-gate readlock_t *readlockp = rwl_entry(rwlp); 890*0Sstevel@tonic-gate if (readlockp->rd_count == 0) { 891*0Sstevel@tonic-gate if (self->ul_error_detection) 892*0Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_unlock", 893*0Sstevel@tonic-gate "readers lock held, " 894*0Sstevel@tonic-gate "but not by the calling thread"); 895*0Sstevel@tonic-gate return (EPERM); 896*0Sstevel@tonic-gate } 897*0Sstevel@tonic-gate /* 898*0Sstevel@tonic-gate * If we hold more than one readers lock on this rwlock, 899*0Sstevel@tonic-gate * just decrement our reference count and return. 900*0Sstevel@tonic-gate */ 901*0Sstevel@tonic-gate if (--readlockp->rd_count != 0) { 902*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 903*0Sstevel@tonic-gate return (0); 904*0Sstevel@tonic-gate } 905*0Sstevel@tonic-gate } else { 906*0Sstevel@tonic-gate /* 907*0Sstevel@tonic-gate * This is a usage error. 908*0Sstevel@tonic-gate * No thread should release an unowned lock. 909*0Sstevel@tonic-gate */ 910*0Sstevel@tonic-gate if (self->ul_error_detection) 911*0Sstevel@tonic-gate rwlock_error(rwlp, "rwlock_unlock", "lock not owned"); 912*0Sstevel@tonic-gate return (EPERM); 913*0Sstevel@tonic-gate } 914*0Sstevel@tonic-gate 915*0Sstevel@tonic-gate if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */ 916*0Sstevel@tonic-gate (void) shared_rwlock_unlock(rwlp, &waked); 917*0Sstevel@tonic-gate } else if (!udp->uberflags.uf_mt) { /* single threaded */ 918*0Sstevel@tonic-gate /* 919*0Sstevel@tonic-gate * In the case of having only a single thread, we don't 920*0Sstevel@tonic-gate * need the protection of queue_lock() (this parallels 921*0Sstevel@tonic-gate * the optimization made in rwlock_lock(), above). 922*0Sstevel@tonic-gate * As in rwlock_lock(), we need to defer signals. 923*0Sstevel@tonic-gate */ 924*0Sstevel@tonic-gate sigoff(self); 925*0Sstevel@tonic-gate if (rwlp->rwlock_readers > 0) { 926*0Sstevel@tonic-gate rwlp->rwlock_readers--; 927*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 928*0Sstevel@tonic-gate } else { 929*0Sstevel@tonic-gate rwlp->rwlock_readers = 0; 930*0Sstevel@tonic-gate /* make it look like we released the embedded mutex */ 931*0Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 932*0Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKCLEAR; 933*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, WRITE_LOCK); 934*0Sstevel@tonic-gate } 935*0Sstevel@tonic-gate sigon(self); 936*0Sstevel@tonic-gate waked = 0; 937*0Sstevel@tonic-gate } else { /* multithreaded */ 938*0Sstevel@tonic-gate queue_head_t *qp; 939*0Sstevel@tonic-gate 940*0Sstevel@tonic-gate qp = queue_lock(rwlp, MX); 941*0Sstevel@tonic-gate if (rwlp->rwlock_readers > 0) { 942*0Sstevel@tonic-gate rwlp->rwlock_readers--; 943*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK); 944*0Sstevel@tonic-gate } else { 945*0Sstevel@tonic-gate rwlp->rwlock_readers = 0; 946*0Sstevel@tonic-gate /* make it look like we released the embedded mutex */ 947*0Sstevel@tonic-gate rwlp->rwlock_mowner = 0; 948*0Sstevel@tonic-gate rwlp->rwlock_mlockw = LOCKCLEAR; 949*0Sstevel@tonic-gate DTRACE_PROBE2(plockstat, rw__release, rwlp, WRITE_LOCK); 950*0Sstevel@tonic-gate } 951*0Sstevel@tonic-gate waked = rw_queue_release(qp, rwlp); 952*0Sstevel@tonic-gate } 953*0Sstevel@tonic-gate 954*0Sstevel@tonic-gate /* 955*0Sstevel@tonic-gate * Yield to the thread we just waked up, just in case we might 956*0Sstevel@tonic-gate * be about to grab the rwlock again immediately upon return. 957*0Sstevel@tonic-gate * This is pretty weak but it helps on a uniprocessor and also 958*0Sstevel@tonic-gate * when cpu affinity has assigned both ourself and the other 959*0Sstevel@tonic-gate * thread to the same CPU. Note that lwp_yield() will yield 960*0Sstevel@tonic-gate * the processor only if the writer is at the same or higher 961*0Sstevel@tonic-gate * priority than ourself. This provides more balanced program 962*0Sstevel@tonic-gate * behavior; it doesn't guarantee acquisition of the lock by 963*0Sstevel@tonic-gate * the pending writer. 964*0Sstevel@tonic-gate */ 965*0Sstevel@tonic-gate if (waked) 966*0Sstevel@tonic-gate lwp_yield(); 967*0Sstevel@tonic-gate return (0); 968*0Sstevel@tonic-gate } 969*0Sstevel@tonic-gate 970*0Sstevel@tonic-gate void 971*0Sstevel@tonic-gate lrw_unlock(rwlock_t *rwlp) 972*0Sstevel@tonic-gate { 973*0Sstevel@tonic-gate (void) __rw_unlock(rwlp); 974*0Sstevel@tonic-gate exit_critical(curthread); 975*0Sstevel@tonic-gate } 976