1/* $OpenBSD: lse.S,v 1.1 2024/07/02 10:25:16 kettenis Exp $ */ 2/* 3 * Copyright (c) 2024 Mark Kettenis <kettenis@openbsd.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18#include <machine/asm.h> 19 20/* 21 * Out-of-line LSE atomics helpers 22 */ 23 24.arch armv8-a+lse 25 26ENTRY(__aarch64_cas4_acq_rel) 27 RETGUARD_SETUP(__aarch64_cas4_acq_rel, x15) 28 adrp x9, arm64_has_lse 29 ldr w9, [x9, :lo12:arm64_has_lse] 30 cbz w9, 1f 31 casal w0, w1, [x2] 32 RETGUARD_CHECK(__aarch64_cas4_acq_rel, x15) 33 ret 341: 35 ldaxr w9, [x2] 36 cmp w9, w0 37 b.ne 2f 38 stlxr w10, w1, [x2] 39 cbnz w10, 1b 402: 41 mov w0, w9 42 RETGUARD_CHECK(__aarch64_cas4_acq_rel, x15) 43 ret 44END(__aarch64_cas4_acq_rel) 45 46ENTRY(__aarch64_cas8_acq_rel) 47 RETGUARD_SETUP(__aarch64_cas8_acq_rel, x15) 48 adrp x9, arm64_has_lse 49 ldr w9, [x9, :lo12:arm64_has_lse] 50 cbz w9, 1f 51 casal x0, x1, [x2] 52 RETGUARD_CHECK(__aarch64_cas8_acq_rel, x15) 53 ret 541: 55 ldaxr x9, [x2] 56 cmp x9, x0 57 b.ne 2f 58 stlxr w10, x1, [x2] 59 cbnz w10, 1b 602: 61 mov x0, x9 62 RETGUARD_CHECK(__aarch64_cas8_acq_rel, x15) 63 ret 64END(__aarch64_cas8_acq_rel) 65 66ENTRY(__aarch64_ldadd4_acq_rel) 67 RETGUARD_SETUP(__aarch64_ldadd4_acq_rel, x15) 68 adrp x9, arm64_has_lse 69 ldr w9, [x9, :lo12:arm64_has_lse] 70 cbz w9, 1f 71 ldaddal w0, w0, [x1] 72 RETGUARD_CHECK(__aarch64_ldadd4_acq_rel, x15) 73 ret 741: 75 ldaxr w9, [x1] 76 add w11, w9, w0 77 stlxr w10, w11, [x1] 78 cbnz w10, 1b 79 mov w0, w9 80 RETGUARD_CHECK(__aarch64_ldadd4_acq_rel, x15) 81 ret 82END(__aarch64_ldadd4_acq_rel) 83 84ENTRY(__aarch64_ldadd8_acq_rel) 85 RETGUARD_SETUP(__aarch64_ldadd8_acq_rel, x15) 86 adrp x9, arm64_has_lse 87 ldr w9, [x9, :lo12:arm64_has_lse] 88 cbz w9, 1f 89 ldaddal x0, x0, [x1] 90 RETGUARD_CHECK(__aarch64_ldadd8_acq_rel, x15) 91 ret 921: 93 ldaxr x9, [x1] 94 add x11, x9, x0 95 stlxr w10, x11, [x1] 96 cbnz w10, 1b 97 mov x0, x9 98 RETGUARD_CHECK(__aarch64_ldadd8_acq_rel, x15) 99 ret 100END(__aarch64_ldadd8_acq_rel) 101 102ENTRY(__aarch64_ldclr4_acq_rel) 103 RETGUARD_SETUP(__aarch64_ldclr4_acq_rel, x15) 104 adrp x9, arm64_has_lse 105 ldr w9, [x9, :lo12:arm64_has_lse] 106 cbz w9, 1f 107 ldclral w0, w0, [x1] 108 RETGUARD_CHECK(__aarch64_ldclr4_acq_rel, x15) 109 ret 1101: 111 ldaxr w9, [x1] 112 bic w11, w9, w0 113 stlxr w10, w11, [x1] 114 cbnz w10, 1b 115 mov w0, w9 116 RETGUARD_CHECK(__aarch64_ldclr4_acq_rel, x15) 117 ret 118END(__aarch64_ldclr4_acq_rel) 119 120ENTRY(__aarch64_ldset4_acq_rel) 121 RETGUARD_SETUP(__aarch64_ldset4_acq_rel, x15) 122 adrp x9, arm64_has_lse 123 ldr w9, [x9, :lo12:arm64_has_lse] 124 cbz w9, 1f 125 ldsetal w0, w0, [x1] 126 RETGUARD_CHECK(__aarch64_ldset4_acq_rel, x15) 127 ret 1281: 129 ldaxr w9, [x1] 130 orr w11, w9, w0 131 stlxr w10, w11, [x1] 132 cbnz w10, 1b 133 mov w0, w9 134 RETGUARD_CHECK(__aarch64_ldset4_acq_rel, x15) 135 ret 136END(__aarch64_ldset4_acq_rel) 137 138ENTRY(__aarch64_swp4_acq_rel) 139 RETGUARD_SETUP(__aarch64_swp4_acq_rel, x15) 140 adrp x9, arm64_has_lse 141 ldr w9, [x9, :lo12:arm64_has_lse] 142 cbz w9, 1f 143 swpal w0, w0, [x1] 144 RETGUARD_CHECK(__aarch64_swp4_acq_rel, x15) 145 ret 1461: 147 ldaxr w9, [x1] 148 stlxr w10, w0, [x1] 149 cbnz w10, 1b 150 mov w0, w9 151 RETGUARD_CHECK(__aarch64_swp4_acq_rel, x15) 152 ret 153END(__aarch64_swp4_acq_rel) 154 155ENTRY(__aarch64_swp8_acq_rel) 156 RETGUARD_SETUP(__aarch64_swp8_acq_rel, x15) 157 adrp x9, arm64_has_lse 158 ldr w9, [x9, :lo12:arm64_has_lse] 159 cbz w9, 1f 160 swpal x0, x0, [x1] 161 RETGUARD_CHECK(__aarch64_swp8_acq_rel, x15) 162 ret 1631: 164 ldaxr x9, [x1] 165 stlxr w10, x0, [x1] 166 cbnz w10, 1b 167 mov x0, x9 168 RETGUARD_CHECK(__aarch64_swp8_acq_rel, x15) 169 ret 170END(__aarch64_swp8_acq_rel) 171