1/* $NetBSD: atomic.S,v 1.15 2009/01/04 18:21:38 pooka Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <machine/asm.h> 33 34#if defined(_KERNEL) && !defined(_RUMPKERNEL) 35#define _HARDKERNEL 36#endif 37 38#ifdef _HARDKERNEL 39#define LOCK(n) .Lpatch/**/n: lock 40#define ALIAS(f, t) STRONG_ALIAS(f,t) 41#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 42#else 43#define LOCK(n) lock 44#define ALIAS(f, t) WEAK_ALIAS(f,t) 45#define ENDLABEL(a) /* nothing */ 46#endif 47 48 .text 49 50ENTRY(_atomic_add_32) 51 movl 4(%esp), %edx 52 movl 8(%esp), %eax 53 LOCK(1) 54 addl %eax, (%edx) 55 ret 56 57ENTRY(_atomic_add_32_nv) 58 movl 4(%esp), %edx 59 movl 8(%esp), %eax 60 movl %eax, %ecx 61 LOCK(2) 62 xaddl %eax, (%edx) 63 addl %ecx, %eax 64 ret 65 66ENTRY(_atomic_and_32) 67 movl 4(%esp), %edx 68 movl 8(%esp), %eax 69 LOCK(3) 70 andl %eax, (%edx) 71 ret 72 73ENTRY(_atomic_and_32_nv) 74 movl 4(%esp), %edx 75 movl (%edx), %eax 760: 77 movl %eax, %ecx 78 andl 8(%esp), %ecx 79 LOCK(4) 80 cmpxchgl %ecx, (%edx) 81 jnz 1f 82 movl %ecx, %eax 83 ret 841: 85 jmp 0b 86 87ENTRY(_atomic_dec_32) 88 movl 4(%esp), %edx 89 LOCK(5) 90 decl (%edx) 91 ret 92 93ENTRY(_atomic_dec_32_nv) 94 movl 4(%esp), %edx 95 movl $-1, %eax 96 LOCK(6) 97 xaddl %eax, (%edx) 98 decl %eax 99 ret 100 101ENTRY(_atomic_inc_32) 102 movl 4(%esp), %edx 103 LOCK(7) 104 incl (%edx) 105 ret 106 107ENTRY(_atomic_inc_32_nv) 108 movl 4(%esp), %edx 109 movl $1, %eax 110 LOCK(8) 111 xaddl %eax, (%edx) 112 incl %eax 113 ret 114 115ENTRY(_atomic_or_32) 116 movl 4(%esp), %edx 117 movl 8(%esp), %eax 118 LOCK(9) 119 orl %eax, (%edx) 120 ret 121 122ENTRY(_atomic_or_32_nv) 123 movl 4(%esp), %edx 124 movl (%edx), %eax 1250: 126 movl %eax, %ecx 127 orl 8(%esp), %ecx 128 LOCK(10) 129 cmpxchgl %ecx, (%edx) 130 jnz 1f 131 movl %ecx, %eax 132 ret 1331: 134 jmp 0b 135 136ENTRY(_atomic_swap_32) 137 movl 4(%esp), %edx 138 movl 8(%esp), %eax 139 xchgl %eax, (%edx) 140 ret 141 142ENTRY(_atomic_cas_32) 143 movl 4(%esp), %edx 144 movl 8(%esp), %eax 145 movl 12(%esp), %ecx 146 LOCK(12) 147 cmpxchgl %ecx, (%edx) 148 /* %eax now contains the old value */ 149 ret 150 151ENTRY(_atomic_cas_32_ni) 152 movl 4(%esp), %edx 153 movl 8(%esp), %eax 154 movl 12(%esp), %ecx 155 cmpxchgl %ecx, (%edx) 156 /* %eax now contains the old value */ 157 ret 158 159ENTRY(_membar_consumer) 160 LOCK(13) 161 addl $0, -4(%esp) 162 ret 163ENDLABEL(membar_consumer_end) 164 165ENTRY(_membar_producer) 166 /* A store is enough */ 167 movl $0, -4(%esp) 168 ret 169ENDLABEL(membar_producer_end) 170 171ENTRY(_membar_sync) 172 LOCK(14) 173 addl $0, -4(%esp) 174 ret 175ENDLABEL(membar_sync_end) 176 177#ifdef _HARDKERNEL 178ENTRY(_atomic_cas_64) 179 pushf 180 cli 181 pushl %edi 182 pushl %ebx 183 movl 12(%esp), %edi 184 movl 16(%esp), %eax 185 movl 20(%esp), %edx 186 movl 24(%esp), %ebx 187 movl 28(%esp), %ecx 188 cmpl 0(%edi), %eax 189 jne 2f 190 cmpl 4(%edi), %edx 191 jne 2f 192 movl %ebx, 0(%edi) 193 movl %ecx, 4(%edi) 1941: 195 popl %ebx 196 popl %edi 197 popf 198 ret 1992: 200 movl 0(%edi), %eax 201 movl 4(%edi), %edx 202 jmp 1b 203ENDLABEL(_atomic_cas_64_end) 204 205ENTRY(_atomic_cas_cx8) 206 pushl %edi 207 pushl %ebx 208 movl 12(%esp), %edi 209 movl 16(%esp), %eax 210 movl 20(%esp), %edx 211 movl 24(%esp), %ebx 212 movl 28(%esp), %ecx 213 LOCK(15) 214 cmpxchg8b (%edi) 215 popl %ebx 216 popl %edi 217 ret 218 .space 32, 0x90 219ENDLABEL(_atomic_cas_cx8_end) 220 221ENTRY(sse2_lfence) 222 lfence 223 ret 224ENDLABEL(sse2_lfence_end) 225 226ENTRY(sse2_mfence) 227 mfence 228 ret 229ENDLABEL(sse2_mfence_end) 230 231atomic_lockpatch: 232 .globl atomic_lockpatch 233 .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 234 .long .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 235 .long .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0 236#else 237ENTRY(_atomic_cas_64) 238 pushl %edi 239 pushl %ebx 240 movl 12(%esp), %edi 241 movl 16(%esp), %eax 242 movl 20(%esp), %edx 243 movl 24(%esp), %ebx 244 movl 28(%esp), %ecx 245 lock 246 cmpxchg8b (%edi) 247 popl %ebx 248 popl %edi 249 ret 250#endif /* _HARDKERNEL */ 251 252ALIAS(atomic_add_32,_atomic_add_32) 253ALIAS(atomic_add_int,_atomic_add_32) 254ALIAS(atomic_add_long,_atomic_add_32) 255ALIAS(atomic_add_ptr,_atomic_add_32) 256 257ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 258ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 259ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 260ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 261 262ALIAS(atomic_and_32,_atomic_and_32) 263ALIAS(atomic_and_uint,_atomic_and_32) 264ALIAS(atomic_and_ulong,_atomic_and_32) 265ALIAS(atomic_and_ptr,_atomic_and_32) 266 267ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 268ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 269ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 270ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 271 272ALIAS(atomic_dec_32,_atomic_dec_32) 273ALIAS(atomic_dec_uint,_atomic_dec_32) 274ALIAS(atomic_dec_ulong,_atomic_dec_32) 275ALIAS(atomic_dec_ptr,_atomic_dec_32) 276 277ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 278ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 279ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 280ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 281 282ALIAS(atomic_inc_32,_atomic_inc_32) 283ALIAS(atomic_inc_uint,_atomic_inc_32) 284ALIAS(atomic_inc_ulong,_atomic_inc_32) 285ALIAS(atomic_inc_ptr,_atomic_inc_32) 286 287ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 288ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 289ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 290ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 291 292ALIAS(atomic_or_32,_atomic_or_32) 293ALIAS(atomic_or_uint,_atomic_or_32) 294ALIAS(atomic_or_ulong,_atomic_or_32) 295ALIAS(atomic_or_ptr,_atomic_or_32) 296 297ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 298ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 299ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 300ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 301 302ALIAS(atomic_swap_32,_atomic_swap_32) 303ALIAS(atomic_swap_uint,_atomic_swap_32) 304ALIAS(atomic_swap_ulong,_atomic_swap_32) 305ALIAS(atomic_swap_ptr,_atomic_swap_32) 306 307ALIAS(atomic_cas_32,_atomic_cas_32) 308ALIAS(atomic_cas_uint,_atomic_cas_32) 309ALIAS(atomic_cas_ulong,_atomic_cas_32) 310ALIAS(atomic_cas_ptr,_atomic_cas_32) 311 312ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 313ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 314ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 315ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 316 317ALIAS(atomic_cas_64,_atomic_cas_64) 318ALIAS(atomic_cas_64_ni,_atomic_cas_64) 319 320ALIAS(membar_consumer,_membar_consumer) 321ALIAS(membar_producer,_membar_producer) 322ALIAS(membar_enter,_membar_consumer) 323ALIAS(membar_exit,_membar_producer) 324ALIAS(membar_sync,_membar_sync) 325 326STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 327STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 328STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 329 330STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 331STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 332STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 333 334STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 335STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 336STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 337 338STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 339STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 340STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 341 342STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 343STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 344STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 345 346STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 347STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 348STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 349 350STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 351STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 352STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 353 354STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 355STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 356STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 357 358STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 359STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 360STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 361 362STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 363STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 364STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 365 366STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 367STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 368STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 369 370STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 371STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 372STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 373 374STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 375STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 376STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 377 378STRONG_ALIAS(_membar_enter,_membar_consumer) 379STRONG_ALIAS(_membar_exit,_membar_producer) 380