1/* $NetBSD: __aarch64_lse.S,v 1.7 2022/08/06 21:31:33 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2021 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nick Hudson. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33 34#include "atomic_op_asm.h" 35 36#if SZ == 1 37#define OPSFX b 38#define R0 w0 39#define R1 w1 40#define R4 w4 41#endif 42 43#if SZ == 2 44#define OPSFX h 45#define R0 w0 46#define R1 w1 47#define R4 w4 48#endif 49 50#if SZ == 4 51#define OPSFX 52#define R0 w0 53#define R1 w1 54#define R4 w4 55#endif 56 57#if SZ == 8 58#define OPSFX 59#define R0 x0 60#define R1 x1 61#define R4 x4 62#endif 63 64#if defined(AR_relax) 65#define ACQ 66#define REL 67#define DMB 68#endif 69 70#if defined(AR_acq) 71#define ACQ a 72#define REL 73#define DMB 74#endif 75 76#if defined(AR_rel) 77#define ACQ 78#define REL l 79#define DMB 80#endif 81 82#if defined(AR_acq_rel) 83#define ACQ a 84#define REL l 85#define DMB 86#endif 87 88#if defined(AR_sync) 89#define ACQ 90#define REL 91#define DMB dmb ish 92#endif 93 94#if defined(OP_clr) 95#define INSNOP bic 96#endif 97 98#if defined(OP_set) 99#define INSNOP orr 100#endif 101 102#if defined(OP_add) 103#define INSNOP add 104#endif 105 106#if defined(OP_eor) 107#define INSNOP eor 108#endif 109 110#define _CONCAT3(A, B, C) __CONCAT3(A,B,C) 111#define _CONCAT4(A, B, C, D) __CONCAT4(A,B,C,D) 112#define _CONCAT5(A, B, C, D, E) __CONCAT5(A,B,C,D,E) 113 114#define FUNC2 _CONCAT3(__aarch64_,OP,AR) 115#define FUNC3 _CONCAT4(__aarch64_,OP,SZ,AR) 116 117#define CASP_FUNC FUNC2 118#define CAS_FUNC FUNC3 119#define SWP_FUNC FUNC3 120#define INSN_FUNC FUNC3 121 122#define LDXR _CONCAT4(ld, ACQ, xr, OPSFX) 123#define STXR _CONCAT4(st, REL, xr, OPSFX) 124#define LDXP _CONCAT3(ld, ACQ, xp) 125#define STXP _CONCAT3(st, REL, xp) 126 127#ifdef _HAVE_LSE 128#define SWP _CONCAT4(swp, ACQ, REL, OPSFX) 129#define CAS _CONCAT4(cas, ACQ, REL, OPSFX) 130#define CASP _CONCAT3(casp, ACQ, REL) 131#define INSN _CONCAT5(ld, OP, ACQ, REL, OPSFX) 132 133 .hidden __aarch64_have_lse_atomics 134 .arch armv8-a+lse 135 136#define DO_LSE_INSN_IF_SUPPORTED(label) \ 137 adrp x4, __aarch64_have_lse_atomics ;\ 138 ldrb w4, [x4, #:lo12:__aarch64_have_lse_atomics] ;\ 139 cbnz w4, label 140 141#endif 142 143#if defined(OP_swp) 144ENTRY_NP(SWP_FUNC) 145#ifdef _HAVE_LSE 146 DO_LSE_INSN_IF_SUPPORTED(99f) 147 DMB 148 SWP R0, R0, [x1] 149 DMB 150 ret 15199: 152#endif 153 mov x4, x0 /* need x0 for return value */ 154 DMB /* potential barrier */ 1551: LDXR R0, [x1] /* load old value */ 156 STXR w3, R4, [x1] /* store new value */ 157 cbnz w3, 2f /* succeed?? no, try again */ 158 DMB /* potential barrier */ 159 ret /* return old value */ 1602: b 1b 161END(SWP_FUNC) 162#endif 163 164#if defined(OP_cas) 165ENTRY_NP(CAS_FUNC) 166#ifdef _HAVE_LSE 167 DO_LSE_INSN_IF_SUPPORTED(99f) 168 DMB 169 CAS R0, R1, [x2] 170 DMB 171 ret 17299: 173#endif 174 mov x4, x0 /* need x0 for return value */ 175 DMB /* potential barrier */ 1761: LDXR R0, [x2] /* load old value */ 177 cmp R0, R4 /* compare */ 178 b.ne 2f /* not equal? return */ 179 STXR w3, R1, [x2] /* store new value */ 180 cbnz w3, 3f /* succeed? nope, try again. */ 181 DMB /* potential barrier */ 1822: ret /* return. */ 1833: b 1b 184END(CAS_FUNC) 185#endif 186 187#if defined(OP_casp) 188ENTRY_NP(CASP_FUNC) 189#ifdef _HAVE_LSE 190 DO_LSE_INSN_IF_SUPPORTED(99f) 191 DMB 192 CASP x0, x1, x2, x3, [x4] 193 DMB 194 ret 19599: 196#endif 197 mov x5, x0 /* need x0 for return value */ 198 mov x6, x1 /* need x1 for return value */ 199 DMB /* potential barrier */ 2001: LDXP x0, x1, [x4] /* load old value */ 201 cmp x0, x5 /* compare */ 202 b.ne 2f /* not equal? return */ 203 cmp x1, x6 204 b.ne 2f /* not equal? return */ 205 STXP w7, x2, x3, [x4] /* store new value */ 206 cbnz w7, 3f /* succeed? nope, try again. */ 207 DMB /* potential barrier */ 2082: ret /* return. */ 2093: b 1b 210END(CASP_FUNC) 211#endif 212 213#if defined(OP_set) || defined(OP_clr) || defined(OP_add) || defined(OP_eor) 214ENTRY_NP(INSN_FUNC) 215#ifdef _HAVE_LSE 216 DO_LSE_INSN_IF_SUPPORTED(99f) 217 DMB 218 INSN R0, R0, [x1] 219 DMB 220 ret 22199: 222#endif 223 mov x4, x0 /* need x0 for return value */ 224 DMB /* potential barrier */ 2251: LDXR R0, [x1] /* load old value */ 226 INSNOP R4, R0, R4 227 STXR w3, R4, [x1] /* store new value */ 228 cbnz w3, 2f /* succeed?? no, try again */ 229 DMB /* potential barrier */ 230 ret /* return old value */ 2312: b 1b 232END(INSN_FUNC) 233#endif 234