1/* $NetBSD: atomic.S,v 1.14 2009/01/12 02:53:29 pooka Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/param.h> 33#include <machine/asm.h> 34 35#ifdef _HARDKERNEL 36#define LOCK(n) .Lpatch/**/n: lock 37#define ALIAS(f, t) STRONG_ALIAS(f,t) 38#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 39#else 40#define LOCK(n) lock 41#define ALIAS(f, t) WEAK_ALIAS(f,t) 42#define ENDLABEL(a) /* nothing */ 43#endif 44 45 .text 46 47/* 32-bit */ 48 49ENTRY(_atomic_add_32) 50 LOCK(1) 51 addl %esi, (%rdi) 52 ret 53 54ENTRY(_atomic_add_32_nv) 55 movl %esi, %eax 56 LOCK(2) 57 xaddl %eax, (%rdi) 58 addl %esi, %eax 59 ret 60 61ENTRY(_atomic_and_32) 62 LOCK(3) 63 andl %esi, (%rdi) 64 ret 65 66ENTRY(_atomic_and_32_nv) 67 movl (%rdi), %eax 681: 69 movl %eax, %ecx 70 andl %esi, %ecx 71 LOCK(4) 72 cmpxchgl %ecx, (%rdi) 73 jnz 1b 74 movl %ecx, %eax 75 ret 76 77ENTRY(_atomic_dec_32) 78 LOCK(5) 79 decl (%rdi) 80 ret 81 82ENTRY(_atomic_dec_32_nv) 83 movl $-1, %eax 84 LOCK(6) 85 xaddl %eax, (%rdi) 86 decl %eax 87 ret 88 89ENTRY(_atomic_inc_32) 90 LOCK(7) 91 incl (%rdi) 92 ret 93 94ENTRY(_atomic_inc_32_nv) 95 movl $1, %eax 96 LOCK(8) 97 xaddl %eax, (%rdi) 98 incl %eax 99 ret 100 101ENTRY(_atomic_or_32) 102 LOCK(9) 103 orl %esi, (%rdi) 104 ret 105 106ENTRY(_atomic_or_32_nv) 107 movl (%rdi), %eax 1081: 109 movl %eax, %ecx 110 orl %esi, %ecx 111 LOCK(10) 112 cmpxchgl %ecx, (%rdi) 113 jnz 1b 114 movl %ecx, %eax 115 ret 116 117ENTRY(_atomic_swap_32) 118 movl %esi, %eax 119 xchgl %eax, (%rdi) 120 ret 121 122ENTRY(_atomic_cas_32) 123 movl %esi, %eax 124 LOCK(12) 125 cmpxchgl %edx, (%rdi) 126 /* %eax now contains the old value */ 127 ret 128 129ENTRY(_atomic_cas_32_ni) 130 movl %esi, %eax 131 cmpxchgl %edx, (%rdi) 132 /* %eax now contains the old value */ 133 ret 134 135/* 64-bit */ 136 137ENTRY(_atomic_add_64) 138 LOCK(13) 139 addq %rsi, (%rdi) 140 ret 141 142ENTRY(_atomic_add_64_nv) 143 movq %rsi, %rax 144 LOCK(14) 145 xaddq %rax, (%rdi) 146 addq %rsi, %rax 147 ret 148 149ENTRY(_atomic_and_64) 150 LOCK(15) 151 andq %rsi, (%rdi) 152 ret 153 154ENTRY(_atomic_and_64_nv) 155 movq (%rdi), %rax 1561: 157 movq %rax, %rcx 158 andq %rsi, %rcx 159 LOCK(16) 160 cmpxchgq %rcx, (%rdi) 161 jnz 1b 162 movq %rcx, %rax 163 ret 164 165ENTRY(_atomic_dec_64) 166 LOCK(17) 167 decq (%rdi) 168 ret 169 170ENTRY(_atomic_dec_64_nv) 171 movq $-1, %rax 172 LOCK(18) 173 xaddq %rax, (%rdi) 174 decq %rax 175 ret 176 177ENTRY(_atomic_inc_64) 178 LOCK(19) 179 incq (%rdi) 180 ret 181 182ENTRY(_atomic_inc_64_nv) 183 movq $1, %rax 184 LOCK(20) 185 xaddq %rax, (%rdi) 186 incq %rax 187 ret 188 189ENTRY(_atomic_or_64) 190 LOCK(21) 191 orq %rsi, (%rdi) 192 ret 193 194ENTRY(_atomic_or_64_nv) 195 movq (%rdi), %rax 1961: 197 movq %rax, %rcx 198 orq %rsi, %rcx 199 LOCK(22) 200 cmpxchgq %rcx, (%rdi) 201 jnz 1b 202 movq %rcx, %rax 203 ret 204 205ENTRY(_atomic_swap_64) 206 movq %rsi, %rax 207 xchgq %rax, (%rdi) 208 ret 209 210ENTRY(_atomic_cas_64) 211 movq %rsi, %rax 212 LOCK(24) 213 cmpxchgq %rdx, (%rdi) 214 /* %eax now contains the old value */ 215 ret 216 217ENTRY(_atomic_cas_64_ni) 218 movq %rsi, %rax 219 cmpxchgq %rdx, (%rdi) 220 /* %eax now contains the old value */ 221 ret 222 223/* memory barriers */ 224 225ENTRY(_membar_consumer) 226 LOCK(25) 227 addq $0, -8(%rsp) 228 ret 229ENDLABEL(membar_consumer_end) 230 231ENTRY(_membar_producer) 232 /* A store is enough */ 233 movq $0, -8(%rsp) 234 ret 235ENDLABEL(membar_producer_end) 236 237ENTRY(_membar_sync) 238 LOCK(26) 239 addq $0, -8(%rsp) 240 ret 241ENDLABEL(membar_sync_end) 242 243#ifdef _HARDKERNEL 244ENTRY(sse2_lfence) 245 lfence 246 ret 247ENDLABEL(sse2_lfence_end) 248 249ENTRY(sse2_mfence) 250 mfence 251 ret 252ENDLABEL(sse2_mfence_end) 253 254atomic_lockpatch: 255 .globl atomic_lockpatch 256 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 257 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 258 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 259 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 260 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 261 .quad .Lpatch26, 0 262#endif /* _HARDKERNEL */ 263 264ALIAS(atomic_add_32,_atomic_add_32) 265ALIAS(atomic_add_64,_atomic_add_64) 266ALIAS(atomic_add_int,_atomic_add_32) 267ALIAS(atomic_add_long,_atomic_add_64) 268ALIAS(atomic_add_ptr,_atomic_add_64) 269 270ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 271ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 272ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 273ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 274ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 275 276ALIAS(atomic_and_32,_atomic_and_32) 277ALIAS(atomic_and_64,_atomic_and_64) 278ALIAS(atomic_and_uint,_atomic_and_32) 279ALIAS(atomic_and_ulong,_atomic_and_64) 280ALIAS(atomic_and_ptr,_atomic_and_64) 281 282ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 283ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 284ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 285ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 286ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 287 288ALIAS(atomic_dec_32,_atomic_dec_32) 289ALIAS(atomic_dec_64,_atomic_dec_64) 290ALIAS(atomic_dec_uint,_atomic_dec_32) 291ALIAS(atomic_dec_ulong,_atomic_dec_64) 292ALIAS(atomic_dec_ptr,_atomic_dec_64) 293 294ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 295ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 296ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 297ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 298ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 299 300ALIAS(atomic_inc_32,_atomic_inc_32) 301ALIAS(atomic_inc_64,_atomic_inc_64) 302ALIAS(atomic_inc_uint,_atomic_inc_32) 303ALIAS(atomic_inc_ulong,_atomic_inc_64) 304ALIAS(atomic_inc_ptr,_atomic_inc_64) 305 306ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 307ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 308ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 309ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 310ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 311 312ALIAS(atomic_or_32,_atomic_or_32) 313ALIAS(atomic_or_uint,_atomic_or_32) 314ALIAS(atomic_or_ulong,_atomic_or_64) 315ALIAS(atomic_or_ptr,_atomic_or_64) 316 317ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 318ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 319ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 320ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 321ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 322 323ALIAS(atomic_swap_32,_atomic_swap_32) 324ALIAS(atomic_swap_64,_atomic_swap_64) 325ALIAS(atomic_swap_uint,_atomic_swap_32) 326ALIAS(atomic_swap_ulong,_atomic_swap_64) 327ALIAS(atomic_swap_ptr,_atomic_swap_64) 328 329ALIAS(atomic_cas_32,_atomic_cas_32) 330ALIAS(atomic_cas_64,_atomic_cas_64) 331ALIAS(atomic_cas_uint,_atomic_cas_32) 332ALIAS(atomic_cas_ulong,_atomic_cas_64) 333ALIAS(atomic_cas_ptr,_atomic_cas_64) 334 335ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 336ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 337ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 338ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 339ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 340 341ALIAS(membar_consumer,_membar_consumer) 342ALIAS(membar_producer,_membar_producer) 343ALIAS(membar_enter,_membar_consumer) 344ALIAS(membar_exit,_membar_producer) 345ALIAS(membar_sync,_membar_sync) 346 347STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 348STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 349STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 350 351STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 352STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 353STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 354 355STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 356STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 357STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 358 359STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 360STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 361STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 362 363STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 364STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 365STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 366 367STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 368STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 369STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 370 371STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 372STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 373STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 374 375STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 376STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 377STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 378 379STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 380STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 381STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 382 383STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 384STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 385STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 386 387STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 388STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 389STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 390 391STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 392STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 393STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 394 395STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 396STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 397STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 398 399STRONG_ALIAS(_membar_enter,_membar_consumer) 400STRONG_ALIAS(_membar_exit,_membar_producer) 401