1/* $NetBSD: atomic.S,v 1.7 2007/12/10 14:13:12 ad Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39#include <machine/asm.h> 40 41#ifdef _KERNEL 42#define LOCK(n) .Lpatch/**/n: lock 43#define ALIAS(f, t) STRONG_ALIAS(f,t) 44#define END(a) _ALIGN_TEXT; LABEL(a) 45#else 46#define LOCK(n) lock 47#define ALIAS(f, t) WEAK_ALIAS(f,t) 48#define END(a) /* nothing */ 49#endif 50 51 .text 52 53/* 32-bit */ 54 55NENTRY(_atomic_add_32) 56 LOCK(1) 57 addl %esi, (%rdi) 58 ret 59 60NENTRY(_atomic_add_32_nv) 61 movl %esi, %eax 62 LOCK(2) 63 xaddl %eax, (%rdi) 64 addl %esi, %eax 65 ret 66 67NENTRY(_atomic_and_32) 68 LOCK(3) 69 andl %esi, (%rdi) 70 ret 71 72NENTRY(_atomic_and_32_nv) 73 movl (%rdi), %eax 741: 75 movl %eax, %ecx 76 andl %esi, %ecx 77 LOCK(4) 78 cmpxchgl %ecx, (%rdi) 79 jnz 1b 80 movl %ecx, %eax 81 ret 82 83NENTRY(_atomic_dec_32) 84 LOCK(5) 85 decl (%rdi) 86 ret 87 88NENTRY(_atomic_dec_32_nv) 89 movl $-1, %eax 90 LOCK(6) 91 xaddl %eax, (%rdi) 92 decl %eax 93 ret 94 95NENTRY(_atomic_inc_32) 96 LOCK(7) 97 incl (%rdi) 98 ret 99 100NENTRY(_atomic_inc_32_nv) 101 movl $1, %eax 102 LOCK(8) 103 xaddl %eax, (%rdi) 104 incl %eax 105 ret 106 107NENTRY(_atomic_or_32) 108 LOCK(9) 109 orl %esi, (%rdi) 110 ret 111 112NENTRY(_atomic_or_32_nv) 113 movl (%rdi), %eax 1141: 115 movl %eax, %ecx 116 orl %esi, %ecx 117 LOCK(10) 118 cmpxchgl %ecx, (%rdi) 119 jnz 1b 120 movl %ecx, %eax 121 ret 122 123NENTRY(_atomic_swap_32) 124 movl %esi, %eax 125 xchgl %eax, (%rdi) 126 ret 127 128NENTRY(_atomic_cas_32) 129 movl %esi, %eax 130 LOCK(12) 131 cmpxchgl %edx, (%rdi) 132 /* %eax now contains the old value */ 133 ret 134 135/* 64-bit */ 136 137NENTRY(_atomic_add_64) 138 LOCK(13) 139 addq %rsi, (%rdi) 140 ret 141 142NENTRY(_atomic_add_64_nv) 143 movq %rsi, %rax 144 LOCK(14) 145 xaddq %rax, (%rdi) 146 addq %rsi, %rax 147 ret 148 149NENTRY(_atomic_and_64) 150 LOCK(15) 151 andq %rsi, (%rdi) 152 ret 153 154NENTRY(_atomic_and_64_nv) 155 movq (%rdi), %rax 1561: 157 movq %rax, %rcx 158 andq %rsi, %rcx 159 LOCK(16) 160 cmpxchgq %rcx, (%rdi) 161 jnz 1b 162 movq %rcx, %rax 163 ret 164 165NENTRY(_atomic_dec_64) 166 LOCK(17) 167 decq (%rdi) 168 ret 169 170NENTRY(_atomic_dec_64_nv) 171 movq $-1, %rax 172 LOCK(18) 173 xaddq %rax, (%rdi) 174 decq %rax 175 ret 176 177NENTRY(_atomic_inc_64) 178 LOCK(19) 179 incq (%rdi) 180 ret 181 182NENTRY(_atomic_inc_64_nv) 183 movq $1, %rax 184 LOCK(20) 185 xaddq %rax, (%rdi) 186 incq %rax 187 ret 188 189NENTRY(_atomic_or_64) 190 LOCK(21) 191 orq %rsi, (%rdi) 192 ret 193 194NENTRY(_atomic_or_64_nv) 195 movq (%rdi), %rax 1961: 197 movq %rax, %rcx 198 orq %rsi, %rcx 199 LOCK(22) 200 cmpxchgq %rcx, (%rdi) 201 jnz 1b 202 movq %rcx, %rax 203 ret 204 205NENTRY(_atomic_swap_64) 206 movq %rsi, %rax 207 xchgq %rax, (%rdi) 208 ret 209 210NENTRY(_atomic_cas_64) 211 movq %rsi, %rax 212 LOCK(24) 213 cmpxchgq %rdx, (%rdi) 214 /* %eax now contains the old value */ 215 ret 216 217/* memory barriers */ 218 219NENTRY(_membar_consumer) 220 LOCK(25) 221 addq $0, -8(%rsp) 222 ret 223END(membar_consumer_end) 224 225NENTRY(_membar_producer) 226 /* A store is enough */ 227 movq $0, -8(%rsp) 228 ret 229END(membar_producer_end) 230 231NENTRY(_membar_enter) 232 /* A store is enough */ 233 movq $0, -8(%rsp) 234 ret 235END(membar_enter_end) 236 237NENTRY(_membar_exit) 238 /* A store is enough */ 239 movq $0, -8(%rsp) 240 ret 241END(membar_exit_end) 242 243NENTRY(_membar_sync) 244 LOCK(26) 245 addq $0, -8(%rsp) 246 ret 247END(membar_sync_end) 248 249#ifdef _KERNEL 250NENTRY(sse2_lfence) 251 lfence 252 ret 253END(sse2_lfence_end) 254 255NENTRY(sse2_mfence) 256 mfence 257 ret 258END(sse2_mfence_end) 259 260atomic_lockpatch: 261 .globl atomic_lockpatch 262 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 263 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 264 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 265 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 266 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 267 .quad .Lpatch26, 0 268#endif /* _KERNEL */ 269 270ALIAS(atomic_add_32,_atomic_add_32) 271ALIAS(atomic_add_64,_atomic_add_64) 272ALIAS(atomic_add_int,_atomic_add_32) 273ALIAS(atomic_add_long,_atomic_add_64) 274ALIAS(atomic_add_ptr,_atomic_add_64) 275 276ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 277ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 278ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 279ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 280ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 281 282ALIAS(atomic_and_32,_atomic_and_32) 283ALIAS(atomic_and_64,_atomic_and_64) 284ALIAS(atomic_and_uint,_atomic_and_32) 285ALIAS(atomic_and_ulong,_atomic_and_64) 286ALIAS(atomic_and_ptr,_atomic_and_64) 287 288ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 289ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 290ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 291ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 292ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 293 294ALIAS(atomic_dec_32,_atomic_dec_32) 295ALIAS(atomic_dec_64,_atomic_dec_64) 296ALIAS(atomic_dec_uint,_atomic_dec_32) 297ALIAS(atomic_dec_ulong,_atomic_dec_64) 298ALIAS(atomic_dec_ptr,_atomic_dec_64) 299 300ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 301ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 302ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 303ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 304ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 305 306ALIAS(atomic_inc_32,_atomic_inc_32) 307ALIAS(atomic_inc_64,_atomic_inc_64) 308ALIAS(atomic_inc_uint,_atomic_inc_32) 309ALIAS(atomic_inc_ulong,_atomic_inc_64) 310ALIAS(atomic_inc_ptr,_atomic_inc_64) 311 312ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 313ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 314ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 315ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 316ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 317 318ALIAS(atomic_or_32,_atomic_or_32) 319ALIAS(atomic_or_uint,_atomic_or_32) 320ALIAS(atomic_or_ulong,_atomic_or_64) 321ALIAS(atomic_or_ptr,_atomic_or_64) 322 323ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 324ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 325ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 326ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 327ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 328 329ALIAS(atomic_swap_32,_atomic_swap_32) 330ALIAS(atomic_swap_64,_atomic_swap_64) 331ALIAS(atomic_swap_uint,_atomic_swap_32) 332ALIAS(atomic_swap_ulong,_atomic_swap_64) 333ALIAS(atomic_swap_ptr,_atomic_swap_64) 334 335ALIAS(atomic_cas_32,_atomic_cas_32) 336ALIAS(atomic_cas_64,_atomic_cas_64) 337ALIAS(atomic_cas_uint,_atomic_cas_32) 338ALIAS(atomic_cas_ulong,_atomic_cas_64) 339ALIAS(atomic_cas_ptr,_atomic_cas_64) 340 341ALIAS(membar_consumer,_membar_consumer) 342ALIAS(membar_producer,_membar_producer) 343ALIAS(membar_enter,_membar_enter) 344ALIAS(membar_exit,_membar_exit) 345ALIAS(membar_sync,_membar_sync) 346 347STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 348STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 349STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 350 351STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 352STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 353STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 354 355STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 356STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 357STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 358 359STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 360STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 361STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 362 363STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 364STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 365STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 366 367STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 368STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 369STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 370 371STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 372STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 373STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 374 375STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 376STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 377STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 378 379STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 380STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 381STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 382 383STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 384STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 385STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 386 387STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 388STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 389STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 390 391STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 392STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 393STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 394