1/* $NetBSD: lock_stubs.s,v 1.14 2022/04/06 22:47:57 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2002, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe and Andrew Doran 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Assembly language lock stubs. These handle the common ("easy") 34 * cases for mutexes, and provide helper routines for rwlocks. 35 * 36 * See sparc/include/mutex.h for additional comments. 37 */ 38 39#include "opt_lockdebug.h" 40 41#include <machine/asm.h> 42#include <machine/param.h> 43 44#include <sparc/sparc/vaddrs.h> 45 46#include <sparc/psl.h> 47 48#include "assym.h" 49 50curlwp = CPUINFO_VA + CPUINFO_CURLWP 51 52/* 53 * Interlock hash table, used by the R/W lock helper routines. 54 */ 55 .section .bss 56 .align 1024 57 .globl _C_LABEL(_lock_hash) 58OTYPE(_C_LABEL(_lock_hash)) 59_C_LABEL(_lock_hash): 60 .space 1024 61 62 .text 63 64#ifndef LOCKDEBUG 65 66/* 67 * void mutex_enter(kmutex_t *mtx); 68 */ 69ENTRY(mutex_enter) 70 sethi %hi(curlwp), %o3 71 ld [%o3 + %lo(curlwp)], %o3 ! current thread 72 ldstub [%o0], %o1 ! try to acquire lock 73mutex_enter_crit_start: .globl mutex_enter_crit_start 74 tst %o1 75 bnz _C_LABEL(mutex_vector_enter) ! nope, hard case 76 77 /* 78 * We now own the lock, but the owner field is not 79 * set. We need to update the lock word with the 80 * our thread pointer. We rely on the fact that the 81 * mutex code will spin or sleep while the mutex is 82 * owned "anonymously". 83 */ 84 sra %o3, 5, %o1 ! curlwp >> 5 85 sethi %hi(0xff000000), %o2 ! finish constructing 86 or %o1, %o2, %o1 ! lock word 87 st %o1, [%o0] 88mutex_enter_crit_end: .globl mutex_enter_crit_end 89 retl 90 nop 91 92/* 93 * void mutex_exit(kmutex_t *mtx); 94 * 95 * Adaptive mutexes on sparc aren't as cheap as other platforms. 96 * However, since we need to test the waiters condition, in the 97 * non-DIAGNOSTIC case we can just clear the owner field. 98 * 99 * NetBSD on sparc uses TSO (total store order), so it's safe to 100 * clear the lock and then test for waiters without worrying about 101 * memory ordering issues. 102 */ 103ENTRY(mutex_exit) 104 sethi %hi(curlwp), %o3 105 ld [%o3 + %lo(curlwp)], %o3 ! current thread 106 sra %o3, 5, %o1 ! curlwp >> 5 107 sethi %hi(0xff000000), %o2 ! finish constructing 108 or %o1, %o2, %o1 ! lock word 109 ld [%o0], %o2 ! get lock word 110 cmp %o1, %o2 ! -> 0 if we own lock 111 bne _C_LABEL(mutex_vector_exit) ! no, hard case 112 nop 113 ldub [%o0 + MTX_LOCK], %o3 ! get has-waiters indicator 114 tst %o3 ! has waiters? 115 bnz _C_LABEL(mutex_wakeup) ! yes, hard case 116 st %g0, [%o0] ! and release lock 117 retl 118 nop 119 120#if 0 /* does not work for MP yet */ 121/* 122 * void mutex_spin_enter(kmutex_t *); 123 */ 124ENTRY(mutex_spin_enter) 125 sethi %hi(CPUINFO_VA+CPUINFO_MTX_COUNT), %o4 126 ld [ %o4 + %lo(CPUINFO_VA+CPUINFO_MTX_COUNT) ], %o5 127 sub %o5, 1, %o1 128 st %o1, [ %o4 + %lo(CPUINFO_VA+CPUINFO_MTX_COUNT) ] 129 ldub [ %o0 + MTX_IPL ], %o2 130 rd %psr, %o1 131 sll %o2, 8, %o2 132 and %o1, PSR_PIL, %o3 133 cmp %o3, %o2 134 bge 1f 135 tst %o5 136 andn %o1, PSR_PIL, %o1 137 wr %o2, %o1, %psr 138 nop 139 nop 140 nop 141 tst %o5 1421: 143 sethi %hi(CPUINFO_VA+CPUINFO_MTX_OLDSPL), %o4 144 bz,a 2f 145 st %o3, [ %o4 + %lo(CPUINFO_VA+CPUINFO_MTX_OLDSPL) ] 1462: 147#if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 148 ldstub [ %o0 + MTX_LOCK ], %o2 149 tst %o2 150 bnz _C_LABEL(mutex_spin_retry) 151 nop 152#endif 153 retl 154 nop 155 156/* 157 * void mutex_spin_exit(kmutex_t *); 158 */ 159ENTRY(mutex_spin_exit) 160 161#if defined(DIAGNOSTIC) 162 ldub [ %o0 + MTX_LOCK ], %o1 163 tst %o1 164 bz _C_LABEL(mutex_vector_exit) 165 nop 166 clrb [ %o0 + MTX_LOCK ] 167#elif defined(MULTIPROCESSOR) 168 clrb [ %o0 + MTX_LOCK ] 169#endif 170 sethi %hi(CPUINFO_VA+CPUINFO_MTX_OLDSPL), %o2 171 ld [ %o2 + %lo(CPUINFO_VA+CPUINFO_MTX_OLDSPL) ], %o3 172 sethi %hi(CPUINFO_VA+CPUINFO_MTX_COUNT), %o2 173 ld [ %o2 + %lo(CPUINFO_VA+CPUINFO_MTX_COUNT) ], %o1 174 addcc %o1, 1, %o4 175 bnz 1f 176 st %o4, [ %o2 + %lo(CPUINFO_VA+CPUINFO_MTX_COUNT) ] 177 rd %psr, %o1 178 andn %o1, PSR_PIL, %o1 179 wr %o3, %o1, %psr 180 nop 181 nop 182 nop 1831: 184 retl 185 nop 186 187#endif 188 189#endif /* LOCKDEBUG */ 190 191/* 192 * int _lock_cas(uintptr_t *ptr, uintptr_t old, uintptr_t new); 193 * 194 * Compare-and-set operation for RW locks. 195 * 196 * XXX On single CPU systems, this should use a restartable sequence: 197 * XXX there we don't need the overhead of interlocking. 198 */ 199ENTRY(_lock_cas) 200 rd %psr, %o4 ! disable interrupts 201 or %o4, PSR_PIL, %o5 202 wr %o5, 0, %psr 203 nop 204 nop 205 nop 206 srl %o0, 3, %o5 207 and %o5, 1023, %o5 208 set _C_LABEL(_lock_hash), %o3 209 add %o5, %o3, %o5 2101: 211 ldstub [%o5], %o3 ! %o5 == interlock address 212 tst %o3 213 bz,a 2f 214 nop 215 nop 216 nop 217 b,a 1b ! spin 218 nop 2192: 220 ld [%o0], %o3 ! lock value 221 cmp %o1, %o3 ! same as expected value? 222 be,a 3f ! yes, store new value 223 st %o2, [%o0] 224 stb %g0, [%o5] 225 wr %o4, 0, %psr ! enable interrupts 226 nop 227 nop 228 nop 229 retl 230 mov %g0, %o0 ! nope 2313: 232 stb %g0, [%o5] 233 wr %o4, 0, %psr ! enable interrupts 234 nop 235 nop 236 nop 237 retl 238 or %g0, 1, %o0 239