1/* $NetBSD: atomic.S,v 1.21 2014/04/22 19:27:17 christos Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/param.h> 33#include <machine/asm.h> 34/* 35 * __HAVE_ constants should not be in <machine/types.h> 36 * because we can't use them from assembly. OTOH we 37 * only need __HAVE_ATOMIC64_OPS here, and we don't. 38 */ 39#ifdef _KERNEL 40#define ALIAS(f, t) STRONG_ALIAS(f,t) 41#else 42#define ALIAS(f, t) WEAK_ALIAS(f,t) 43#endif 44 45#ifdef _HARDKERNEL 46#define LOCK(n) .Lpatch ## n: lock 47#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 48#else 49#define LOCK(n) lock 50#define ENDLABEL(a) /* nothing */ 51#endif 52 53 .text 54 55ENTRY(_atomic_add_32) 56 movl 4(%esp), %edx 57 movl 8(%esp), %eax 58 LOCK(1) 59 addl %eax, (%edx) 60 ret 61 62ENTRY(_atomic_add_32_nv) 63 movl 4(%esp), %edx 64 movl 8(%esp), %eax 65 movl %eax, %ecx 66 LOCK(2) 67 xaddl %eax, (%edx) 68 addl %ecx, %eax 69 ret 70 71ENTRY(_atomic_and_32) 72 movl 4(%esp), %edx 73 movl 8(%esp), %eax 74 LOCK(3) 75 andl %eax, (%edx) 76 ret 77 78ENTRY(_atomic_and_32_nv) 79 movl 4(%esp), %edx 80 movl (%edx), %eax 810: 82 movl %eax, %ecx 83 andl 8(%esp), %ecx 84 LOCK(4) 85 cmpxchgl %ecx, (%edx) 86 jnz 1f 87 movl %ecx, %eax 88 ret 891: 90 jmp 0b 91 92ENTRY(_atomic_dec_32) 93 movl 4(%esp), %edx 94 LOCK(5) 95 decl (%edx) 96 ret 97 98ENTRY(_atomic_dec_32_nv) 99 movl 4(%esp), %edx 100 movl $-1, %eax 101 LOCK(6) 102 xaddl %eax, (%edx) 103 decl %eax 104 ret 105 106ENTRY(_atomic_inc_32) 107 movl 4(%esp), %edx 108 LOCK(7) 109 incl (%edx) 110 ret 111 112ENTRY(_atomic_inc_32_nv) 113 movl 4(%esp), %edx 114 movl $1, %eax 115 LOCK(8) 116 xaddl %eax, (%edx) 117 incl %eax 118 ret 119 120ENTRY(_atomic_or_32) 121 movl 4(%esp), %edx 122 movl 8(%esp), %eax 123 LOCK(9) 124 orl %eax, (%edx) 125 ret 126 127ENTRY(_atomic_or_32_nv) 128 movl 4(%esp), %edx 129 movl (%edx), %eax 1300: 131 movl %eax, %ecx 132 orl 8(%esp), %ecx 133 LOCK(10) 134 cmpxchgl %ecx, (%edx) 135 jnz 1f 136 movl %ecx, %eax 137 ret 1381: 139 jmp 0b 140 141ENTRY(_atomic_swap_32) 142 movl 4(%esp), %edx 143 movl 8(%esp), %eax 144 xchgl %eax, (%edx) 145 ret 146 147ENTRY(_atomic_cas_32) 148 movl 4(%esp), %edx 149 movl 8(%esp), %eax 150 movl 12(%esp), %ecx 151 LOCK(12) 152 cmpxchgl %ecx, (%edx) 153 /* %eax now contains the old value */ 154 ret 155 156ENTRY(_atomic_cas_32_ni) 157 movl 4(%esp), %edx 158 movl 8(%esp), %eax 159 movl 12(%esp), %ecx 160 cmpxchgl %ecx, (%edx) 161 /* %eax now contains the old value */ 162 ret 163 164ENTRY(_membar_consumer) 165 LOCK(13) 166 addl $0, -4(%esp) 167 ret 168ENDLABEL(membar_consumer_end) 169 170ENTRY(_membar_producer) 171 /* A store is enough */ 172 movl $0, -4(%esp) 173 ret 174ENDLABEL(membar_producer_end) 175 176ENTRY(_membar_sync) 177 LOCK(14) 178 addl $0, -4(%esp) 179 ret 180ENDLABEL(membar_sync_end) 181 182#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 183ENTRY(_atomic_cas_64) 184#ifdef _HARDKERNEL 185 pushf 186 cli 187#endif /* _HARDKERNEL */ 188 pushl %edi 189 pushl %ebx 190 movl 12(%esp), %edi 191 movl 16(%esp), %eax 192 movl 20(%esp), %edx 193 movl 24(%esp), %ebx 194 movl 28(%esp), %ecx 195 cmpl 0(%edi), %eax 196 jne 2f 197 cmpl 4(%edi), %edx 198 jne 2f 199 movl %ebx, 0(%edi) 200 movl %ecx, 4(%edi) 2011: 202 popl %ebx 203 popl %edi 204#ifdef _HARDKERNEL 205 popf 206#endif /* _HARDKERNEL */ 207 ret 2082: 209 movl 0(%edi), %eax 210 movl 4(%edi), %edx 211 jmp 1b 212ENDLABEL(_atomic_cas_64_end) 213 214ENTRY(_atomic_cas_cx8) 215 pushl %edi 216 pushl %ebx 217 movl 12(%esp), %edi 218 movl 16(%esp), %eax 219 movl 20(%esp), %edx 220 movl 24(%esp), %ebx 221 movl 28(%esp), %ecx 222 LOCK(15) 223 cmpxchg8b (%edi) 224 popl %ebx 225 popl %edi 226 ret 227#ifdef _HARDKERNEL 228#ifdef GPROF 229 .space 16, 0x90 230#else 231 .space 32, 0x90 232#endif 233#endif /* _HARDKERNEL */ 234ENDLABEL(_atomic_cas_cx8_end) 235#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 236 237#ifdef _HARDKERNEL 238ENTRY(sse2_lfence) 239 lfence 240 ret 241ENDLABEL(sse2_lfence_end) 242 243ENTRY(sse2_mfence) 244 mfence 245 ret 246ENDLABEL(sse2_mfence_end) 247 248atomic_lockpatch: 249 .globl atomic_lockpatch 250 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 251 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 252 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 253#endif /* _HARDKERNEL */ 254 255ALIAS(atomic_add_32,_atomic_add_32) 256ALIAS(atomic_add_int,_atomic_add_32) 257ALIAS(atomic_add_long,_atomic_add_32) 258ALIAS(atomic_add_ptr,_atomic_add_32) 259 260ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 261ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 262ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 263ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 264 265ALIAS(atomic_and_32,_atomic_and_32) 266ALIAS(atomic_and_uint,_atomic_and_32) 267ALIAS(atomic_and_ulong,_atomic_and_32) 268ALIAS(atomic_and_ptr,_atomic_and_32) 269 270ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 271ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 272ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 273ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 274 275ALIAS(atomic_dec_32,_atomic_dec_32) 276ALIAS(atomic_dec_uint,_atomic_dec_32) 277ALIAS(atomic_dec_ulong,_atomic_dec_32) 278ALIAS(atomic_dec_ptr,_atomic_dec_32) 279 280ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 281ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 282ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 283ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 284 285ALIAS(atomic_inc_32,_atomic_inc_32) 286ALIAS(atomic_inc_uint,_atomic_inc_32) 287ALIAS(atomic_inc_ulong,_atomic_inc_32) 288ALIAS(atomic_inc_ptr,_atomic_inc_32) 289 290ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 291ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 292ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 293ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 294 295ALIAS(atomic_or_32,_atomic_or_32) 296ALIAS(atomic_or_uint,_atomic_or_32) 297ALIAS(atomic_or_ulong,_atomic_or_32) 298ALIAS(atomic_or_ptr,_atomic_or_32) 299 300ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 301ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 302ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 303ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 304 305ALIAS(atomic_swap_32,_atomic_swap_32) 306ALIAS(atomic_swap_uint,_atomic_swap_32) 307ALIAS(atomic_swap_ulong,_atomic_swap_32) 308ALIAS(atomic_swap_ptr,_atomic_swap_32) 309 310ALIAS(atomic_cas_32,_atomic_cas_32) 311ALIAS(atomic_cas_uint,_atomic_cas_32) 312ALIAS(atomic_cas_ulong,_atomic_cas_32) 313ALIAS(atomic_cas_ptr,_atomic_cas_32) 314 315ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 316ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 317ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 318ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 319 320#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 321ALIAS(atomic_cas_64,_atomic_cas_64) 322ALIAS(atomic_cas_64_ni,_atomic_cas_64) 323ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) 324#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 325 326ALIAS(membar_consumer,_membar_consumer) 327ALIAS(membar_producer,_membar_producer) 328ALIAS(membar_enter,_membar_consumer) 329ALIAS(membar_exit,_membar_producer) 330ALIAS(membar_sync,_membar_sync) 331 332STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 333STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 334STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 335 336STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 337STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 338STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 339 340STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 341STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 342STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 343 344STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 345STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 346STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 347 348STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 349STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 350STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 351 352STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 353STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 354STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 355 356STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 357STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 358STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 359 360STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 361STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 362STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 363 364STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 365STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 366STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 367 368STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 369STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 370STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 371 372STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 373STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 374STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 375 376STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 377STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 378STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 379 380STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 381STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 382STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 383 384STRONG_ALIAS(_membar_enter,_membar_consumer) 385STRONG_ALIAS(_membar_exit,_membar_producer) 386