1 /* $OpenBSD: rthread_rwlock.c,v 1.13 2019/03/03 18:39:10 visa Exp $ */ 2 /* 3 * Copyright (c) 2019 Martin Pieuchot <mpi@openbsd.org> 4 * Copyright (c) 2012 Philip Guenther <guenther@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <stdlib.h> 20 #include <unistd.h> 21 #include <errno.h> 22 23 #include <pthread.h> 24 25 #include "rthread.h" 26 #include "synch.h" 27 28 #define UNLOCKED 0 29 #define MAXREADER 0x7ffffffe 30 #define WRITER 0x7fffffff 31 #define WAITING 0x80000000 32 #define COUNT(v) ((v) & WRITER) 33 34 #define SPIN_COUNT 128 35 #if defined(__i386__) || defined(__amd64__) 36 #define SPIN_WAIT() asm volatile("pause": : : "memory") 37 #else 38 #define SPIN_WAIT() do { } while (0) 39 #endif 40 41 static _atomic_lock_t rwlock_init_lock = _SPINLOCK_UNLOCKED; 42 43 int 44 pthread_rwlock_init(pthread_rwlock_t *lockp, 45 const pthread_rwlockattr_t *attrp __unused) 46 { 47 pthread_rwlock_t rwlock; 48 49 rwlock = calloc(1, sizeof(*rwlock)); 50 if (!rwlock) 51 return (errno); 52 53 *lockp = rwlock; 54 55 return (0); 56 } 57 DEF_STD(pthread_rwlock_init); 58 59 int 60 pthread_rwlock_destroy(pthread_rwlock_t *lockp) 61 { 62 pthread_rwlock_t rwlock; 63 64 rwlock = *lockp; 65 if (rwlock) { 66 if (rwlock->value != UNLOCKED) { 67 #define MSG "pthread_rwlock_destroy on rwlock with waiters!\n" 68 write(2, MSG, sizeof(MSG) - 1); 69 #undef MSG 70 return (EBUSY); 71 } 72 free((void *)rwlock); 73 *lockp = NULL; 74 } 75 76 return (0); 77 } 78 79 static int 80 _rthread_rwlock_ensure_init(pthread_rwlock_t *rwlockp) 81 { 82 int ret = 0; 83 84 /* 85 * If the rwlock is statically initialized, perform the dynamic 86 * initialization. 87 */ 88 if (*rwlockp == NULL) { 89 _spinlock(&rwlock_init_lock); 90 if (*rwlockp == NULL) 91 ret = pthread_rwlock_init(rwlockp, NULL); 92 _spinunlock(&rwlock_init_lock); 93 } 94 return (ret); 95 } 96 97 static int 98 _rthread_rwlock_tryrdlock(pthread_rwlock_t rwlock) 99 { 100 unsigned int val; 101 102 do { 103 val = rwlock->value; 104 if (COUNT(val) == WRITER) 105 return (EBUSY); 106 if (COUNT(val) == MAXREADER) 107 return (EAGAIN); 108 } while (atomic_cas_uint(&rwlock->value, val, val + 1) != val); 109 110 membar_enter_after_atomic(); 111 return (0); 112 } 113 114 static int 115 _rthread_rwlock_timedrdlock(pthread_rwlock_t *rwlockp, int trywait, 116 const struct timespec *abs, int timed) 117 { 118 pthread_t self = pthread_self(); 119 pthread_rwlock_t rwlock; 120 unsigned int val, new; 121 int i, error; 122 123 if ((error = _rthread_rwlock_ensure_init(rwlockp))) 124 return (error); 125 126 rwlock = *rwlockp; 127 _rthread_debug(5, "%p: rwlock_%srdlock %p (%u)\n", self, 128 (timed ? "timed" : (trywait ? "try" : "")), (void *)rwlock, 129 rwlock->value); 130 131 error = _rthread_rwlock_tryrdlock(rwlock); 132 if (error != EBUSY || trywait) 133 return (error); 134 135 /* Try hard to not enter the kernel. */ 136 for (i = 0; i < SPIN_COUNT; i++) { 137 val = rwlock->value; 138 if (val == UNLOCKED || (val & WAITING)) 139 break; 140 141 SPIN_WAIT(); 142 } 143 144 while ((error = _rthread_rwlock_tryrdlock(rwlock)) == EBUSY) { 145 val = rwlock->value; 146 if (val == UNLOCKED || (COUNT(val)) != WRITER) 147 continue; 148 new = val | WAITING; 149 if (atomic_cas_uint(&rwlock->value, val, new) == val) { 150 error = _twait(&rwlock->value, new, CLOCK_REALTIME, 151 abs); 152 } 153 if (error == ETIMEDOUT) 154 break; 155 } 156 157 return (error); 158 159 } 160 161 int 162 pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlockp) 163 { 164 return (_rthread_rwlock_timedrdlock(rwlockp, 1, NULL, 0)); 165 } 166 167 int 168 pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlockp, 169 const struct timespec *abs) 170 { 171 return (_rthread_rwlock_timedrdlock(rwlockp, 0, abs, 1)); 172 } 173 174 int 175 pthread_rwlock_rdlock(pthread_rwlock_t *rwlockp) 176 { 177 return (_rthread_rwlock_timedrdlock(rwlockp, 0, NULL, 0)); 178 } 179 180 static int 181 _rthread_rwlock_tryrwlock(pthread_rwlock_t rwlock) 182 { 183 if (atomic_cas_uint(&rwlock->value, UNLOCKED, WRITER) != UNLOCKED) 184 return (EBUSY); 185 186 membar_enter_after_atomic(); 187 return (0); 188 } 189 190 191 static int 192 _rthread_rwlock_timedwrlock(pthread_rwlock_t *rwlockp, int trywait, 193 const struct timespec *abs, int timed) 194 { 195 pthread_t self = pthread_self(); 196 pthread_rwlock_t rwlock; 197 unsigned int val, new; 198 int i, error; 199 200 if ((error = _rthread_rwlock_ensure_init(rwlockp))) 201 return (error); 202 203 rwlock = *rwlockp; 204 _rthread_debug(5, "%p: rwlock_%swrlock %p (%u)\n", self, 205 (timed ? "timed" : (trywait ? "try" : "")), (void *)rwlock, 206 rwlock->value); 207 208 error = _rthread_rwlock_tryrwlock(rwlock); 209 if (error != EBUSY || trywait) 210 return (error); 211 212 /* Try hard to not enter the kernel. */ 213 for (i = 0; i < SPIN_COUNT; i++) { 214 val = rwlock->value; 215 if (val == UNLOCKED || (val & WAITING)) 216 break; 217 218 SPIN_WAIT(); 219 } 220 221 while ((error = _rthread_rwlock_tryrwlock(rwlock)) == EBUSY) { 222 val = rwlock->value; 223 if (val == UNLOCKED) 224 continue; 225 new = val | WAITING; 226 if (atomic_cas_uint(&rwlock->value, val, new) == val) { 227 error = _twait(&rwlock->value, new, CLOCK_REALTIME, 228 abs); 229 } 230 if (error == ETIMEDOUT) 231 break; 232 } 233 234 return (error); 235 } 236 237 int 238 pthread_rwlock_trywrlock(pthread_rwlock_t *rwlockp) 239 { 240 return (_rthread_rwlock_timedwrlock(rwlockp, 1, NULL, 0)); 241 } 242 243 int 244 pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlockp, 245 const struct timespec *abs) 246 { 247 return (_rthread_rwlock_timedwrlock(rwlockp, 0, abs, 1)); 248 } 249 250 int 251 pthread_rwlock_wrlock(pthread_rwlock_t *rwlockp) 252 { 253 return (_rthread_rwlock_timedwrlock(rwlockp, 0, NULL, 0)); 254 } 255 256 int 257 pthread_rwlock_unlock(pthread_rwlock_t *rwlockp) 258 { 259 pthread_t self = pthread_self(); 260 pthread_rwlock_t rwlock; 261 unsigned int val, new; 262 263 rwlock = *rwlockp; 264 _rthread_debug(5, "%p: rwlock_unlock %p\n", self, (void *)rwlock); 265 266 membar_exit_before_atomic(); 267 do { 268 val = rwlock->value; 269 if (COUNT(val) == WRITER || COUNT(val) == 1) 270 new = UNLOCKED; 271 else 272 new = val - 1; 273 } while (atomic_cas_uint(&rwlock->value, val, new) != val); 274 275 if (new == UNLOCKED && (val & WAITING)) 276 _wake(&rwlock->value, INT_MAX); 277 278 return (0); 279 } 280