1/* $NetBSD: atomic.S,v 1.5 2007/12/09 17:38:51 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39#include <machine/asm.h> 40 41#ifdef _KERNEL 42#define LOCK(n) .Lpatch/**/n: lock 43#define ALIAS(f, t) STRONG_ALIAS(f,t) 44#define END(a) _ALIGN_TEXT; LABEL(a) 45#else 46#define LOCK(n) lock 47#define ALIAS(f, t) WEAK_ALIAS(f,t) 48#define END(a) /* nothing */ 49#endif 50 51 .text 52 53NENTRY(_atomic_add_32) 54 movl 4(%esp), %edx 55 movl 8(%esp), %eax 56 LOCK(1) 57 addl %eax, (%edx) 58 ret 59 60NENTRY(_atomic_add_32_nv) 61 movl 4(%esp), %edx 62 movl 8(%esp), %eax 63 movl %eax, %ecx 64 LOCK(2) 65 xaddl %eax, (%edx) 66 addl %ecx, %eax 67 ret 68 69NENTRY(_atomic_and_32) 70 movl 4(%esp), %edx 71 movl 8(%esp), %eax 72 LOCK(3) 73 andl %eax, (%edx) 74 ret 75 76NENTRY(_atomic_and_32_nv) 77 movl 4(%esp), %edx 78 movl (%edx), %eax 791: 80 movl %eax, %ecx 81 andl 8(%esp), %ecx 82 LOCK(4) 83 cmpxchgl %ecx, (%edx) 84 jnz 1b 85 movl %ecx, %eax 86 ret 87 88NENTRY(_atomic_dec_32) 89 movl 4(%esp), %edx 90 LOCK(5) 91 decl (%edx) 92 ret 93 94NENTRY(_atomic_dec_32_nv) 95 movl 4(%esp), %edx 96 movl $-1, %eax 97 LOCK(6) 98 xaddl %eax, (%edx) 99 decl %eax 100 ret 101 102NENTRY(_atomic_inc_32) 103 movl 4(%esp), %edx 104 LOCK(7) 105 incl (%edx) 106 ret 107 108NENTRY(_atomic_inc_32_nv) 109 movl 4(%esp), %edx 110 movl $1, %eax 111 LOCK(8) 112 xaddl %eax, (%edx) 113 incl %eax 114 ret 115 116NENTRY(_atomic_or_32) 117 movl 4(%esp), %edx 118 movl 8(%esp), %eax 119 LOCK(9) 120 orl %eax, (%edx) 121 ret 122 123NENTRY(_atomic_or_32_nv) 124 movl 4(%esp), %edx 125 movl (%edx), %eax 1261: 127 movl %eax, %ecx 128 orl 8(%esp), %ecx 129 LOCK(10) 130 cmpxchgl %ecx, (%edx) 131 jnz 1b 132 movl %ecx, %eax 133 ret 134 135NENTRY(_atomic_swap_32) 136 movl 4(%esp), %edx 137 movl 8(%esp), %eax 138 xchgl %eax, (%edx) 139 ret 140 141NENTRY(_atomic_cas_32) 142 movl 4(%esp), %edx 143 movl 8(%esp), %eax 144 movl 12(%esp), %ecx 145 LOCK(12) 146 cmpxchgl %ecx, (%edx) 147 /* %eax now contains the old value */ 148 ret 149 150NENTRY(_membar_consumer) 151 LOCK(13) 152 addl $0, -4(%esp) 153 ret 154END(membar_consumer_end) 155 156NENTRY(_membar_producer) 157 /* A store is enough */ 158 movl $0, -4(%esp) 159 ret 160END(membar_producer_end) 161 162NENTRY(_membar_enter) 163 /* A store is enough */ 164 movl $0, -4(%esp) 165 ret 166END(membar_enter_end) 167 168NENTRY(_membar_exit) 169 /* A store is enough */ 170 movl $0, -4(%esp) 171 ret 172END(membar_exit_end) 173 174NENTRY(_membar_sync) 175 LOCK(14) 176 addl $0, -4(%esp) 177 ret 178END(membar_sync_end) 179 180#ifdef _KERNEL 181NENTRY(sse2_lfence) 182 lfence 183 ret 184END(sse2_lfence_end) 185 186NENTRY(sse2_mfence) 187 mfence 188 ret 189END(sse2_mfence_end) 190 191atomic_lockpatch: 192 .globl atomic_lockpatch 193 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 194 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 195 .long .Lpatch12, .Lpatch13, .Lpatch14, 0 196#endif /* _KERNEL */ 197 198ALIAS(atomic_add_32,_atomic_add_32) 199ALIAS(atomic_add_int,_atomic_add_32) 200ALIAS(atomic_add_long,_atomic_add_32) 201ALIAS(atomic_add_ptr,_atomic_add_32) 202 203ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 204ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 205ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 206ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 207 208ALIAS(atomic_and_32,_atomic_and_32) 209ALIAS(atomic_and_uint,_atomic_and_32) 210ALIAS(atomic_and_ulong,_atomic_and_32) 211ALIAS(atomic_and_ptr,_atomic_and_32) 212 213ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 214ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 215ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 216ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 217 218ALIAS(atomic_dec_32,_atomic_dec_32) 219ALIAS(atomic_dec_uint,_atomic_dec_32) 220ALIAS(atomic_dec_ulong,_atomic_dec_32) 221ALIAS(atomic_dec_ptr,_atomic_dec_32) 222 223ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 224ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 225ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 226ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 227 228ALIAS(atomic_inc_32,_atomic_inc_32) 229ALIAS(atomic_inc_uint,_atomic_inc_32) 230ALIAS(atomic_inc_ulong,_atomic_inc_32) 231ALIAS(atomic_inc_ptr,_atomic_inc_32) 232 233ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 234ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 235ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 236ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 237 238ALIAS(atomic_or_32,_atomic_or_32) 239ALIAS(atomic_or_uint,_atomic_or_32) 240ALIAS(atomic_or_ulong,_atomic_or_32) 241ALIAS(atomic_or_ptr,_atomic_or_32) 242 243ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 244ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 245ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 246ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 247 248ALIAS(atomic_swap_32,_atomic_swap_32) 249ALIAS(atomic_swap_uint,_atomic_swap_32) 250ALIAS(atomic_swap_ulong,_atomic_swap_32) 251ALIAS(atomic_swap_ptr,_atomic_swap_32) 252 253ALIAS(atomic_cas_32,_atomic_cas_32) 254ALIAS(atomic_cas_uint,_atomic_cas_32) 255ALIAS(atomic_cas_ulong,_atomic_cas_32) 256ALIAS(atomic_cas_ptr,_atomic_cas_32) 257 258ALIAS(membar_consumer,_membar_consumer) 259ALIAS(membar_producer,_membar_producer) 260ALIAS(membar_enter,_membar_enter) 261ALIAS(membar_exit,_membar_exit) 262ALIAS(membar_sync,_membar_sync) 263 264STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 265STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 266STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 267 268STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 269STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 270STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 271 272STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 273STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 274STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 275 276STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 277STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 278STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 279 280STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 281STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 282STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 283 284STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 285STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 286STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 287 288STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 289STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 290STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 291 292STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 293STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 294STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 295 296STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 297STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 298STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 299 300STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 301STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 302STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 303 304STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 305STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 306STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 307 308STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 309STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 310STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 311