1/* $NetBSD: atomic.S,v 1.17 2014/05/22 15:23:11 uebayasi Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/param.h> 33#include <machine/asm.h> 34 35#ifdef _KERNEL 36#define ALIAS(f, t) STRONG_ALIAS(f,t) 37#else 38#define ALIAS(f, t) WEAK_ALIAS(f,t) 39#endif 40 41#ifdef _HARDKERNEL 42#define LOCK(n) .Lpatch ## n: lock 43#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a) 44#else 45#define LOCK(n) lock 46#define ENDLABEL(a) /* nothing */ 47#endif 48 49 .text 50 51/* 32-bit */ 52 53ENTRY(_atomic_add_32) 54 LOCK(1) 55 addl %esi, (%rdi) 56 ret 57END(_atomic_add_32) 58 59ENTRY(_atomic_add_32_nv) 60 movl %esi, %eax 61 LOCK(2) 62 xaddl %eax, (%rdi) 63 addl %esi, %eax 64 ret 65END(_atomic_add_32_nv) 66 67ENTRY(_atomic_and_32) 68 LOCK(3) 69 andl %esi, (%rdi) 70 ret 71END(_atomic_and_32) 72 73ENTRY(_atomic_and_32_nv) 74 movl (%rdi), %eax 751: 76 movl %eax, %ecx 77 andl %esi, %ecx 78 LOCK(4) 79 cmpxchgl %ecx, (%rdi) 80 jnz 1b 81 movl %ecx, %eax 82 ret 83END(_atomic_and_32_nv) 84 85ENTRY(_atomic_dec_32) 86 LOCK(5) 87 decl (%rdi) 88 ret 89END(_atomic_dec_32) 90 91ENTRY(_atomic_dec_32_nv) 92 movl $-1, %eax 93 LOCK(6) 94 xaddl %eax, (%rdi) 95 decl %eax 96 ret 97END(_atomic_dec_32_nv) 98 99ENTRY(_atomic_inc_32) 100 LOCK(7) 101 incl (%rdi) 102 ret 103END(_atomic_inc_32) 104 105ENTRY(_atomic_inc_32_nv) 106 movl $1, %eax 107 LOCK(8) 108 xaddl %eax, (%rdi) 109 incl %eax 110 ret 111END(_atomic_inc_32_nv) 112 113ENTRY(_atomic_or_32) 114 LOCK(9) 115 orl %esi, (%rdi) 116 ret 117END(_atomic_or_32) 118 119ENTRY(_atomic_or_32_nv) 120 movl (%rdi), %eax 1211: 122 movl %eax, %ecx 123 orl %esi, %ecx 124 LOCK(10) 125 cmpxchgl %ecx, (%rdi) 126 jnz 1b 127 movl %ecx, %eax 128 ret 129END(_atomic_or_32_nv) 130 131ENTRY(_atomic_swap_32) 132 movl %esi, %eax 133 xchgl %eax, (%rdi) 134 ret 135END(_atomic_swap_32) 136 137ENTRY(_atomic_cas_32) 138 movl %esi, %eax 139 LOCK(12) 140 cmpxchgl %edx, (%rdi) 141 /* %eax now contains the old value */ 142 ret 143END(_atomic_cas_32) 144 145ENTRY(_atomic_cas_32_ni) 146 movl %esi, %eax 147 cmpxchgl %edx, (%rdi) 148 /* %eax now contains the old value */ 149 ret 150END(_atomic_cas_32_ni) 151 152/* 64-bit */ 153 154ENTRY(_atomic_add_64) 155 LOCK(13) 156 addq %rsi, (%rdi) 157 ret 158END(_atomic_add_64) 159 160ENTRY(_atomic_add_64_nv) 161 movq %rsi, %rax 162 LOCK(14) 163 xaddq %rax, (%rdi) 164 addq %rsi, %rax 165 ret 166END(_atomic_add_64_nv) 167 168ENTRY(_atomic_and_64) 169 LOCK(15) 170 andq %rsi, (%rdi) 171 ret 172END(_atomic_and_64) 173 174ENTRY(_atomic_and_64_nv) 175 movq (%rdi), %rax 1761: 177 movq %rax, %rcx 178 andq %rsi, %rcx 179 LOCK(16) 180 cmpxchgq %rcx, (%rdi) 181 jnz 1b 182 movq %rcx, %rax 183 ret 184END(_atomic_and_64_nv) 185 186ENTRY(_atomic_dec_64) 187 LOCK(17) 188 decq (%rdi) 189 ret 190END(_atomic_dec_64) 191 192ENTRY(_atomic_dec_64_nv) 193 movq $-1, %rax 194 LOCK(18) 195 xaddq %rax, (%rdi) 196 decq %rax 197 ret 198END(_atomic_dec_64_nv) 199 200ENTRY(_atomic_inc_64) 201 LOCK(19) 202 incq (%rdi) 203 ret 204END(_atomic_inc_64) 205 206ENTRY(_atomic_inc_64_nv) 207 movq $1, %rax 208 LOCK(20) 209 xaddq %rax, (%rdi) 210 incq %rax 211 ret 212END(_atomic_inc_64_nv) 213 214ENTRY(_atomic_or_64) 215 LOCK(21) 216 orq %rsi, (%rdi) 217 ret 218END(_atomic_or_64) 219 220ENTRY(_atomic_or_64_nv) 221 movq (%rdi), %rax 2221: 223 movq %rax, %rcx 224 orq %rsi, %rcx 225 LOCK(22) 226 cmpxchgq %rcx, (%rdi) 227 jnz 1b 228 movq %rcx, %rax 229 ret 230END(_atomic_or_64_nv) 231 232ENTRY(_atomic_swap_64) 233 movq %rsi, %rax 234 xchgq %rax, (%rdi) 235 ret 236END(_atomic_swap_64) 237 238ENTRY(_atomic_cas_64) 239 movq %rsi, %rax 240 LOCK(24) 241 cmpxchgq %rdx, (%rdi) 242 /* %eax now contains the old value */ 243 ret 244END(_atomic_cas_64) 245 246ENTRY(_atomic_cas_64_ni) 247 movq %rsi, %rax 248 cmpxchgq %rdx, (%rdi) 249 /* %eax now contains the old value */ 250 ret 251END(_atomic_cas_64_ni) 252 253/* memory barriers */ 254 255ENTRY(_membar_consumer) 256 LOCK(25) 257 addq $0, -8(%rsp) 258 ret 259END(_membar_consumer) 260ENDLABEL(membar_consumer_end) 261 262ENTRY(_membar_producer) 263 /* A store is enough */ 264 movq $0, -8(%rsp) 265 ret 266END(_membar_producer) 267ENDLABEL(membar_producer_end) 268 269ENTRY(_membar_sync) 270 LOCK(26) 271 addq $0, -8(%rsp) 272 ret 273END(_membar_sync) 274ENDLABEL(membar_sync_end) 275 276#ifdef _HARDKERNEL 277ENTRY(sse2_lfence) 278 lfence 279 ret 280END(sse2_lfence) 281ENDLABEL(sse2_lfence_end) 282 283ENTRY(sse2_mfence) 284 mfence 285 ret 286END(sse2_mfence) 287ENDLABEL(sse2_mfence_end) 288 289atomic_lockpatch: 290 .globl atomic_lockpatch 291 .quad .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5 292 .quad .Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10 293 .quad .Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15 294 .quad .Lpatch16, .Lpatch17, .Lpatch18, .Lpatch19, .Lpatch20 295 .quad .Lpatch21, .Lpatch22, .Lpatch24, .Lpatch25 296 .quad .Lpatch26, 0 297#endif /* _HARDKERNEL */ 298 299ALIAS(atomic_add_32,_atomic_add_32) 300ALIAS(atomic_add_64,_atomic_add_64) 301ALIAS(atomic_add_int,_atomic_add_32) 302ALIAS(atomic_add_long,_atomic_add_64) 303ALIAS(atomic_add_ptr,_atomic_add_64) 304 305ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 306ALIAS(atomic_add_64_nv,_atomic_add_64_nv) 307ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 308ALIAS(atomic_add_long_nv,_atomic_add_64_nv) 309ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) 310 311ALIAS(atomic_and_32,_atomic_and_32) 312ALIAS(atomic_and_64,_atomic_and_64) 313ALIAS(atomic_and_uint,_atomic_and_32) 314ALIAS(atomic_and_ulong,_atomic_and_64) 315ALIAS(atomic_and_ptr,_atomic_and_64) 316 317ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 318ALIAS(atomic_and_64_nv,_atomic_and_64_nv) 319ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 320ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) 321ALIAS(atomic_and_ptr_nv,_atomic_and_64_nv) 322 323ALIAS(atomic_dec_32,_atomic_dec_32) 324ALIAS(atomic_dec_64,_atomic_dec_64) 325ALIAS(atomic_dec_uint,_atomic_dec_32) 326ALIAS(atomic_dec_ulong,_atomic_dec_64) 327ALIAS(atomic_dec_ptr,_atomic_dec_64) 328 329ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 330ALIAS(atomic_dec_64_nv,_atomic_dec_64_nv) 331ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 332ALIAS(atomic_dec_ulong_nv,_atomic_dec_64_nv) 333ALIAS(atomic_dec_ptr_nv,_atomic_dec_64_nv) 334 335ALIAS(atomic_inc_32,_atomic_inc_32) 336ALIAS(atomic_inc_64,_atomic_inc_64) 337ALIAS(atomic_inc_uint,_atomic_inc_32) 338ALIAS(atomic_inc_ulong,_atomic_inc_64) 339ALIAS(atomic_inc_ptr,_atomic_inc_64) 340 341ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 342ALIAS(atomic_inc_64_nv,_atomic_inc_64_nv) 343ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 344ALIAS(atomic_inc_ulong_nv,_atomic_inc_64_nv) 345ALIAS(atomic_inc_ptr_nv,_atomic_inc_64_nv) 346 347ALIAS(atomic_or_32,_atomic_or_32) 348ALIAS(atomic_or_uint,_atomic_or_32) 349ALIAS(atomic_or_ulong,_atomic_or_64) 350ALIAS(atomic_or_ptr,_atomic_or_64) 351 352ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 353ALIAS(atomic_or_64_nv,_atomic_or_64_nv) 354ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 355ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) 356ALIAS(atomic_or_ptr_nv,_atomic_or_64_nv) 357 358ALIAS(atomic_swap_32,_atomic_swap_32) 359ALIAS(atomic_swap_64,_atomic_swap_64) 360ALIAS(atomic_swap_uint,_atomic_swap_32) 361ALIAS(atomic_swap_ulong,_atomic_swap_64) 362ALIAS(atomic_swap_ptr,_atomic_swap_64) 363 364ALIAS(atomic_cas_32,_atomic_cas_32) 365ALIAS(atomic_cas_64,_atomic_cas_64) 366ALIAS(atomic_cas_uint,_atomic_cas_32) 367ALIAS(atomic_cas_ulong,_atomic_cas_64) 368ALIAS(atomic_cas_ptr,_atomic_cas_64) 369 370ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 371ALIAS(atomic_cas_64_ni,_atomic_cas_64_ni) 372ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 373ALIAS(atomic_cas_ulong_ni,_atomic_cas_64_ni) 374ALIAS(atomic_cas_ptr_ni,_atomic_cas_64_ni) 375 376ALIAS(membar_consumer,_membar_consumer) 377ALIAS(membar_producer,_membar_producer) 378ALIAS(membar_enter,_membar_consumer) 379ALIAS(membar_exit,_membar_producer) 380ALIAS(membar_sync,_membar_sync) 381 382STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 383STRONG_ALIAS(_atomic_add_long,_atomic_add_64) 384STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) 385 386STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 387STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) 388STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) 389 390STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 391STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) 392STRONG_ALIAS(_atomic_and_ptr,_atomic_and_64) 393 394STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 395STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) 396STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_64_nv) 397 398STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 399STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) 400STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) 401 402STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 403STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_64_nv) 404STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_64_nv) 405 406STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 407STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) 408STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) 409 410STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 411STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_64_nv) 412STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_64_nv) 413 414STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 415STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) 416STRONG_ALIAS(_atomic_or_ptr,_atomic_or_64) 417 418STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 419STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) 420STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_64_nv) 421 422STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 423STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) 424STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) 425 426STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 427STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) 428STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) 429 430STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 431STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64_ni) 432STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64_ni) 433 434STRONG_ALIAS(_membar_enter,_membar_consumer) 435STRONG_ALIAS(_membar_exit,_membar_producer) 436