1 /* $NetBSD: linux_rwsem.c,v 1.4 2021/12/19 11:21:54 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2021 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: linux_rwsem.c,v 1.4 2021/12/19 11:21:54 riastradh Exp $"); 31 32 #include <sys/types.h> 33 34 #include <sys/condvar.h> 35 #include <sys/lwp.h> 36 #include <sys/lockdebug.h> 37 #include <sys/rwlock.h> 38 39 #include <machine/limits.h> 40 41 #include <lib/libkern/libkern.h> 42 43 #include <linux/rwsem.h> 44 45 #define RWSEM_WANTLOCK(RWSEM) \ 46 LOCKDEBUG_WANTLOCK((RWSEM)->rws_debug, (RWSEM), \ 47 (uintptr_t)__builtin_return_address(0), 0) 48 #define RWSEM_LOCKED_EX(RWSEM) \ 49 LOCKDEBUG_LOCKED((RWSEM)->rws_debug, (RWSEM), NULL, \ 50 (uintptr_t)__builtin_return_address(0), 0) 51 #define RWSEM_LOCKED_SH(RWSEM) \ 52 LOCKDEBUG_LOCKED((RWSEM)->rws_debug, (RWSEM), NULL, \ 53 (uintptr_t)__builtin_return_address(0), 1) 54 #define RWSEM_UNLOCKED_EX(RWSEM) \ 55 LOCKDEBUG_UNLOCKED((RWSEM)->rws_debug, (RWSEM), \ 56 (uintptr_t)__builtin_return_address(0), 0) 57 #define RWSEM_UNLOCKED_SH(RWSEM) \ 58 LOCKDEBUG_UNLOCKED((RWSEM)->rws_debug, (RWSEM), \ 59 (uintptr_t)__builtin_return_address(0), 1) 60 61 #ifdef LOCKDEBUG 62 static void 63 rwsem_dump(const volatile void *cookie, lockop_printer_t pr) 64 { 65 const volatile struct rw_semaphore *rwsem = cookie; 66 67 pr("%-13s: %p", "writer", rwsem->rws_writer); 68 pr("%-13s: %u", "readers", rwsem->rws_readers); 69 pr("%-13s: %s", "writewanted", rwsem->rws_writewanted ? "yes" : "no"); 70 } 71 72 static lockops_t rwsem_lockops = { 73 .lo_name = "Linux read/write semaphore", 74 .lo_type = LOCKOPS_SLEEP, 75 .lo_dump = rwsem_dump, 76 }; 77 #endif 78 79 void 80 init_rwsem(struct rw_semaphore *rwsem) 81 { 82 83 mutex_init(&rwsem->rws_lock, MUTEX_DEFAULT, IPL_VM); 84 cv_init(&rwsem->rws_cv, "lnxrwsem"); 85 rwsem->rws_writer = NULL; 86 rwsem->rws_readers = 0; 87 88 #ifdef LOCKDEBUG 89 rwsem->rws_debug = LOCKDEBUG_ALLOC(rwsem, &rwsem_lockops, 90 (uintptr_t)__builtin_return_address(0)); 91 #endif 92 } 93 94 void 95 destroy_rwsem(struct rw_semaphore *rwsem) 96 { 97 98 KASSERT(rwsem->rws_readers == 0); 99 KASSERT(rwsem->rws_writer == NULL); 100 101 #ifdef LOCKDEBUG 102 LOCKDEBUG_FREE(rwsem->rws_debug, rwsem); 103 #endif 104 105 cv_destroy(&rwsem->rws_cv); 106 mutex_destroy(&rwsem->rws_lock); 107 } 108 109 void 110 down_read(struct rw_semaphore *rwsem) 111 { 112 113 RWSEM_WANTLOCK(rwsem); 114 115 mutex_enter(&rwsem->rws_lock); 116 while (rwsem->rws_writer || rwsem->rws_writewanted) 117 cv_wait(&rwsem->rws_cv, &rwsem->rws_lock); 118 KASSERT(rwsem->rws_readers < UINT_MAX); 119 rwsem->rws_readers++; 120 mutex_exit(&rwsem->rws_lock); 121 122 RWSEM_LOCKED_SH(rwsem); 123 } 124 125 bool 126 down_read_trylock(struct rw_semaphore *rwsem) 127 { 128 bool ret = false; 129 130 /* 131 * Note: Linux apparently relies on down_read_trylock to 132 * quietly succeed when the caller already holds a reader lock. 133 * This is why we can't use rwlock(9), which absolutely 134 * prohibits recursive use and crashes immediately under 135 * LOCKDEBUG if you try it. 136 */ 137 138 mutex_enter(&rwsem->rws_lock); 139 if (rwsem->rws_writer == NULL && !rwsem->rws_writewanted) { 140 KASSERT(rwsem->rws_readers < UINT_MAX); 141 rwsem->rws_readers++; 142 ret = true; 143 } 144 mutex_exit(&rwsem->rws_lock); 145 146 if (ret) { 147 RWSEM_LOCKED_SH(rwsem); 148 } 149 150 return ret; 151 } 152 153 void 154 up_read(struct rw_semaphore *rwsem) 155 { 156 157 RWSEM_UNLOCKED_SH(rwsem); 158 159 mutex_enter(&rwsem->rws_lock); 160 KASSERT(rwsem->rws_readers); 161 KASSERT(rwsem->rws_writer == NULL); 162 if (--rwsem->rws_readers == 0) 163 cv_broadcast(&rwsem->rws_cv); 164 mutex_exit(&rwsem->rws_lock); 165 } 166 167 void 168 down_write(struct rw_semaphore *rwsem) 169 { 170 171 RWSEM_WANTLOCK(rwsem); 172 173 mutex_enter(&rwsem->rws_lock); 174 175 /* If another writer is waiting, get in the queue. */ 176 while (rwsem->rws_writewanted) 177 cv_wait(&rwsem->rws_cv, &rwsem->rws_lock); 178 179 /* 180 * No other writers waiting. Our turn. Announce our intent to 181 * readers, and wait for the writer or readers to finish. 182 */ 183 rwsem->rws_writewanted = true; 184 while (rwsem->rws_writer || rwsem->rws_readers) { 185 KASSERTMSG(rwsem->rws_writer != curlwp, 186 "locking against myself: rwsem=%p lwp=%p", rwsem, curlwp); 187 cv_wait(&rwsem->rws_cv, &rwsem->rws_lock); 188 } 189 190 /* At last, it is ours! */ 191 KASSERT(rwsem->rws_readers == 0); 192 KASSERT(rwsem->rws_writer == NULL); 193 KASSERT(rwsem->rws_writewanted); 194 rwsem->rws_writewanted = false; 195 rwsem->rws_writer = curlwp; 196 197 mutex_exit(&rwsem->rws_lock); 198 199 RWSEM_LOCKED_EX(rwsem); 200 } 201 202 void 203 up_write(struct rw_semaphore *rwsem) 204 { 205 206 RWSEM_UNLOCKED_EX(rwsem); 207 208 mutex_enter(&rwsem->rws_lock); 209 KASSERT(rwsem->rws_writer == curlwp); 210 KASSERT(rwsem->rws_readers == 0); 211 rwsem->rws_writer = NULL; 212 cv_broadcast(&rwsem->rws_cv); 213 mutex_exit(&rwsem->rws_lock); 214 } 215 216 void 217 downgrade_write(struct rw_semaphore *rwsem) 218 { 219 220 RWSEM_UNLOCKED_EX(rwsem); 221 222 mutex_enter(&rwsem->rws_lock); 223 KASSERT(rwsem->rws_writer == curlwp); 224 KASSERT(rwsem->rws_readers == 0); 225 rwsem->rws_writer = NULL; 226 rwsem->rws_readers = 1; 227 cv_broadcast(&rwsem->rws_cv); 228 mutex_exit(&rwsem->rws_lock); 229 230 RWSEM_LOCKED_SH(rwsem); 231 } 232