1/* $NetBSD: atomic.S,v 1.29 2020/05/01 08:32:50 maxv Exp $ */ 2 3/*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/param.h> 33#include <machine/asm.h> 34/* 35 * __HAVE_ constants should not be in <machine/types.h> 36 * because we can't use them from assembly. OTOH we 37 * only need __HAVE_ATOMIC64_OPS here, and we don't. 38 */ 39#ifdef _KERNEL 40#define ALIAS(f, t) STRONG_ALIAS(f,t) 41#else 42#define ALIAS(f, t) WEAK_ALIAS(f,t) 43#endif 44 45#ifdef _HARDKERNEL 46#include "opt_xen.h" 47#include <machine/frameasm.h> 48#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock 49#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 7); 50#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7); 51#define HOTPATCH_CAS_64 HOTPATCH(HP_NAME_CAS_64, 49); 52#else 53#define LOCK lock 54#define HOTPATCH_SSE2_LFENCE /* nothing */ 55#define HOTPATCH_SSE2_MFENCE /* nothing */ 56#define HOTPATCH_CAS_64 /* nothing */ 57#endif 58 59 .text 60 61ENTRY(_atomic_add_32) 62 movl 4(%esp), %edx 63 movl 8(%esp), %eax 64 LOCK 65 addl %eax, (%edx) 66 ret 67END(_atomic_add_32) 68 69ENTRY(_atomic_add_32_nv) 70 movl 4(%esp), %edx 71 movl 8(%esp), %eax 72 movl %eax, %ecx 73 LOCK 74 xaddl %eax, (%edx) 75 addl %ecx, %eax 76 ret 77END(_atomic_add_32_nv) 78 79ENTRY(_atomic_and_32) 80 movl 4(%esp), %edx 81 movl 8(%esp), %eax 82 LOCK 83 andl %eax, (%edx) 84 ret 85END(_atomic_and_32) 86 87ENTRY(_atomic_and_32_nv) 88 movl 4(%esp), %edx 89 movl (%edx), %eax 900: 91 movl %eax, %ecx 92 andl 8(%esp), %ecx 93 LOCK 94 cmpxchgl %ecx, (%edx) 95 jnz 1f 96 movl %ecx, %eax 97 ret 981: 99 jmp 0b 100END(_atomic_and_32_nv) 101 102ENTRY(_atomic_dec_32) 103 movl 4(%esp), %edx 104 LOCK 105 decl (%edx) 106 ret 107END(_atomic_dec_32) 108 109ENTRY(_atomic_dec_32_nv) 110 movl 4(%esp), %edx 111 movl $-1, %eax 112 LOCK 113 xaddl %eax, (%edx) 114 decl %eax 115 ret 116END(_atomic_dec_32_nv) 117 118ENTRY(_atomic_inc_32) 119 movl 4(%esp), %edx 120 LOCK 121 incl (%edx) 122 ret 123END(_atomic_inc_32) 124 125ENTRY(_atomic_inc_32_nv) 126 movl 4(%esp), %edx 127 movl $1, %eax 128 LOCK 129 xaddl %eax, (%edx) 130 incl %eax 131 ret 132END(_atomic_inc_32_nv) 133 134ENTRY(_atomic_or_32) 135 movl 4(%esp), %edx 136 movl 8(%esp), %eax 137 LOCK 138 orl %eax, (%edx) 139 ret 140END(_atomic_or_32) 141 142ENTRY(_atomic_or_32_nv) 143 movl 4(%esp), %edx 144 movl (%edx), %eax 1450: 146 movl %eax, %ecx 147 orl 8(%esp), %ecx 148 LOCK 149 cmpxchgl %ecx, (%edx) 150 jnz 1f 151 movl %ecx, %eax 152 ret 1531: 154 jmp 0b 155END(_atomic_or_32_nv) 156 157ENTRY(_atomic_swap_32) 158 movl 4(%esp), %edx 159 movl 8(%esp), %eax 160 xchgl %eax, (%edx) 161 ret 162END(_atomic_swap_32) 163 164ENTRY(_atomic_cas_32) 165 movl 4(%esp), %edx 166 movl 8(%esp), %eax 167 movl 12(%esp), %ecx 168 LOCK 169 cmpxchgl %ecx, (%edx) 170 /* %eax now contains the old value */ 171 ret 172END(_atomic_cas_32) 173 174ENTRY(_atomic_cas_32_ni) 175 movl 4(%esp), %edx 176 movl 8(%esp), %eax 177 movl 12(%esp), %ecx 178 cmpxchgl %ecx, (%edx) 179 /* %eax now contains the old value */ 180 ret 181END(_atomic_cas_32_ni) 182 183ENTRY(_membar_consumer) 184 HOTPATCH_SSE2_LFENCE 185 /* 7 bytes of instructions */ 186 LOCK 187 addl $0, -4(%esp) 188 ret 189END(_membar_consumer) 190 191ENTRY(_membar_producer) 192 /* A store is enough */ 193 movl $0, -4(%esp) 194 ret 195END(_membar_producer) 196 197ENTRY(_membar_sync) 198 HOTPATCH_SSE2_MFENCE 199 /* 7 bytes of instructions */ 200 LOCK 201 addl $0, -4(%esp) 202 ret 203END(_membar_sync) 204 205#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 206#ifdef XENPV 207STRONG_ALIAS(_atomic_cas_64,_atomic_cas_cx8) 208#else 209ENTRY(_atomic_cas_64) 210 HOTPATCH_CAS_64 211 /* 49 bytes of instructions */ 212#ifdef _HARDKERNEL 213 pushf 214 cli 215#endif 216 pushl %edi 217 pushl %ebx 218 movl 12(%esp), %edi 219 movl 16(%esp), %eax 220 movl 20(%esp), %edx 221 movl 24(%esp), %ebx 222 movl 28(%esp), %ecx 223 cmpl 0(%edi), %eax 224 jne 2f 225 cmpl 4(%edi), %edx 226 jne 2f 227 movl %ebx, 0(%edi) 228 movl %ecx, 4(%edi) 2291: 230 popl %ebx 231 popl %edi 232#ifdef _HARDKERNEL 233 popf 234#endif 235 ret 2362: 237 movl 0(%edi), %eax 238 movl 4(%edi), %edx 239 jmp 1b 240END(_atomic_cas_64) 241#endif /* !XENPV */ 242 243ENTRY(_atomic_cas_cx8) 244 /* 49 bytes of instructions */ 245 pushl %edi 246 pushl %ebx 247 movl 12(%esp), %edi 248 movl 16(%esp), %eax 249 movl 20(%esp), %edx 250 movl 24(%esp), %ebx 251 movl 28(%esp), %ecx 252 LOCK 253 cmpxchg8b (%edi) 254 popl %ebx 255 popl %edi 256 ret 257#ifdef _HARDKERNEL 258 .space 20, 0xCC 259#endif 260END(_atomic_cas_cx8) 261LABEL(_atomic_cas_cx8_end) 262#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 263 264ALIAS(atomic_add_32,_atomic_add_32) 265ALIAS(atomic_add_int,_atomic_add_32) 266ALIAS(atomic_add_long,_atomic_add_32) 267ALIAS(atomic_add_ptr,_atomic_add_32) 268 269ALIAS(atomic_add_32_nv,_atomic_add_32_nv) 270ALIAS(atomic_add_int_nv,_atomic_add_32_nv) 271ALIAS(atomic_add_long_nv,_atomic_add_32_nv) 272ALIAS(atomic_add_ptr_nv,_atomic_add_32_nv) 273 274ALIAS(atomic_and_32,_atomic_and_32) 275ALIAS(atomic_and_uint,_atomic_and_32) 276ALIAS(atomic_and_ulong,_atomic_and_32) 277ALIAS(atomic_and_ptr,_atomic_and_32) 278 279ALIAS(atomic_and_32_nv,_atomic_and_32_nv) 280ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) 281ALIAS(atomic_and_ulong_nv,_atomic_and_32_nv) 282ALIAS(atomic_and_ptr_nv,_atomic_and_32_nv) 283 284ALIAS(atomic_dec_32,_atomic_dec_32) 285ALIAS(atomic_dec_uint,_atomic_dec_32) 286ALIAS(atomic_dec_ulong,_atomic_dec_32) 287ALIAS(atomic_dec_ptr,_atomic_dec_32) 288 289ALIAS(atomic_dec_32_nv,_atomic_dec_32_nv) 290ALIAS(atomic_dec_uint_nv,_atomic_dec_32_nv) 291ALIAS(atomic_dec_ulong_nv,_atomic_dec_32_nv) 292ALIAS(atomic_dec_ptr_nv,_atomic_dec_32_nv) 293 294ALIAS(atomic_inc_32,_atomic_inc_32) 295ALIAS(atomic_inc_uint,_atomic_inc_32) 296ALIAS(atomic_inc_ulong,_atomic_inc_32) 297ALIAS(atomic_inc_ptr,_atomic_inc_32) 298 299ALIAS(atomic_inc_32_nv,_atomic_inc_32_nv) 300ALIAS(atomic_inc_uint_nv,_atomic_inc_32_nv) 301ALIAS(atomic_inc_ulong_nv,_atomic_inc_32_nv) 302ALIAS(atomic_inc_ptr_nv,_atomic_inc_32_nv) 303 304ALIAS(atomic_or_32,_atomic_or_32) 305ALIAS(atomic_or_uint,_atomic_or_32) 306ALIAS(atomic_or_ulong,_atomic_or_32) 307ALIAS(atomic_or_ptr,_atomic_or_32) 308 309ALIAS(atomic_or_32_nv,_atomic_or_32_nv) 310ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) 311ALIAS(atomic_or_ulong_nv,_atomic_or_32_nv) 312ALIAS(atomic_or_ptr_nv,_atomic_or_32_nv) 313 314ALIAS(atomic_swap_32,_atomic_swap_32) 315ALIAS(atomic_swap_uint,_atomic_swap_32) 316ALIAS(atomic_swap_ulong,_atomic_swap_32) 317ALIAS(atomic_swap_ptr,_atomic_swap_32) 318 319ALIAS(atomic_cas_32,_atomic_cas_32) 320ALIAS(atomic_cas_uint,_atomic_cas_32) 321ALIAS(atomic_cas_ulong,_atomic_cas_32) 322ALIAS(atomic_cas_ptr,_atomic_cas_32) 323 324ALIAS(atomic_cas_32_ni,_atomic_cas_32_ni) 325ALIAS(atomic_cas_uint_ni,_atomic_cas_32_ni) 326ALIAS(atomic_cas_ulong_ni,_atomic_cas_32_ni) 327ALIAS(atomic_cas_ptr_ni,_atomic_cas_32_ni) 328 329#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL) 330ALIAS(atomic_cas_64,_atomic_cas_64) 331ALIAS(atomic_cas_64_ni,_atomic_cas_64) 332ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) 333#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */ 334 335ALIAS(membar_consumer,_membar_consumer) 336ALIAS(membar_producer,_membar_producer) 337ALIAS(membar_enter,_membar_consumer) 338ALIAS(membar_exit,_membar_producer) 339ALIAS(membar_sync,_membar_sync) 340 341STRONG_ALIAS(_atomic_add_int,_atomic_add_32) 342STRONG_ALIAS(_atomic_add_long,_atomic_add_32) 343STRONG_ALIAS(_atomic_add_ptr,_atomic_add_32) 344 345STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) 346STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_32_nv) 347STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_32_nv) 348 349STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) 350STRONG_ALIAS(_atomic_and_ulong,_atomic_and_32) 351STRONG_ALIAS(_atomic_and_ptr,_atomic_and_32) 352 353STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) 354STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_32_nv) 355STRONG_ALIAS(_atomic_and_ptr_nv,_atomic_and_32_nv) 356 357STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) 358STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_32) 359STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_32) 360 361STRONG_ALIAS(_atomic_dec_uint_nv,_atomic_dec_32_nv) 362STRONG_ALIAS(_atomic_dec_ulong_nv,_atomic_dec_32_nv) 363STRONG_ALIAS(_atomic_dec_ptr_nv,_atomic_dec_32_nv) 364 365STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) 366STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_32) 367STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_32) 368 369STRONG_ALIAS(_atomic_inc_uint_nv,_atomic_inc_32_nv) 370STRONG_ALIAS(_atomic_inc_ulong_nv,_atomic_inc_32_nv) 371STRONG_ALIAS(_atomic_inc_ptr_nv,_atomic_inc_32_nv) 372 373STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) 374STRONG_ALIAS(_atomic_or_ulong,_atomic_or_32) 375STRONG_ALIAS(_atomic_or_ptr,_atomic_or_32) 376 377STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) 378STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_32_nv) 379STRONG_ALIAS(_atomic_or_ptr_nv,_atomic_or_32_nv) 380 381STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) 382STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_32) 383STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_32) 384 385STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) 386STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_32) 387STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_32) 388 389STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32_ni) 390STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_32_ni) 391STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_32_ni) 392 393STRONG_ALIAS(_membar_enter,_membar_consumer) 394STRONG_ALIAS(_membar_exit,_membar_producer) 395 396#ifdef _HARDKERNEL 397 .section .rodata 398 399LABEL(sse2_lfence) 400 lfence 401 ret 402 nop; nop; nop; 403LABEL(sse2_lfence_end) 404 405LABEL(sse2_mfence) 406 mfence 407 ret 408 nop; nop; nop; 409LABEL(sse2_mfence_end) 410#endif /* _HARDKERNEL */ 411