1 /* Linux-specific atomic operations for ARM EABI. 2 Copyright (C) 2008-2020 Free Software Foundation, Inc. 3 Contributed by CodeSourcery. 4 5 This file is part of GCC. 6 7 GCC is free software; you can redistribute it and/or modify it under 8 the terms of the GNU General Public License as published by the Free 9 Software Foundation; either version 3, or (at your option) any later 10 version. 11 12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or 14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 15 for more details. 16 17 Under Section 7 of GPL version 3, you are granted additional 18 permissions described in the GCC Runtime Library Exception, version 19 3.1, as published by the Free Software Foundation. 20 21 You should have received a copy of the GNU General Public License and 22 a copy of the GCC Runtime Library Exception along with this program; 23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24 <http://www.gnu.org/licenses/>. */ 25 26 /* Kernel helper for compare-and-exchange. */ 27 typedef int (__kernel_cmpxchg_t) (int oldval, int newval, int *ptr); 28 29 #define STR(X) #X 30 #define XSTR(X) STR(X) 31 32 #define KERNEL_CMPXCHG 0xffff0fc0 33 34 #if __FDPIC__ 35 /* Non-FDPIC ABIs call __kernel_cmpxchg directly by dereferencing its 36 address, but under FDPIC we would generate a broken call 37 sequence. That's why we have to implement __kernel_cmpxchg and 38 __kernel_dmb here: this way, the FDPIC call sequence works. */ 39 #define __kernel_cmpxchg __fdpic_cmpxchg 40 #else 41 #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *) KERNEL_CMPXCHG) 42 #endif 43 44 /* Kernel helper for memory barrier. */ 45 typedef void (__kernel_dmb_t) (void); 46 47 #define KERNEL_DMB 0xffff0fa0 48 49 #if __FDPIC__ 50 #define __kernel_dmb __fdpic_dmb 51 #else 52 #define __kernel_dmb (*(__kernel_dmb_t *) KERNEL_DMB) 53 #endif 54 55 #if __FDPIC__ 56 static int __fdpic_cmpxchg (int oldval, int newval, int *ptr) 57 { 58 int result; 59 60 asm volatile ( 61 "ldr ip, 1f\n\t" 62 "bx ip\n\t" 63 "1:\n\t" 64 ".word " XSTR(KERNEL_CMPXCHG) "\n\t" 65 : "=r" (result) 66 : "r" (oldval) , "r" (newval), "r" (ptr) 67 : "r3", "memory"); 68 /* The result is actually returned by the kernel helper, we need 69 this to avoid a warning. */ 70 return result; 71 } 72 73 static void __fdpic_dmb (void) 74 { 75 asm volatile ( 76 "ldr ip, 1f\n\t" 77 "bx ip\n\t" 78 "1:\n\t" 79 ".word " XSTR(KERNEL_DMB) "\n\t" 80 ); 81 } 82 83 #endif 84 85 /* Note: we implement byte, short and int versions of atomic operations using 86 the above kernel helpers; see linux-atomic-64bit.c for "long long" (64-bit) 87 operations. */ 88 89 #define HIDDEN __attribute__ ((visibility ("hidden"))) 90 91 #ifdef __ARMEL__ 92 #define INVERT_MASK_1 0 93 #define INVERT_MASK_2 0 94 #else 95 #define INVERT_MASK_1 24 96 #define INVERT_MASK_2 16 97 #endif 98 99 #define MASK_1 0xffu 100 #define MASK_2 0xffffu 101 102 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \ 103 int HIDDEN \ 104 __sync_fetch_and_##OP##_4 (int *ptr, int val) \ 105 { \ 106 int failure, tmp; \ 107 \ 108 do { \ 109 tmp = *ptr; \ 110 failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr); \ 111 } while (failure != 0); \ 112 \ 113 return tmp; \ 114 } 115 116 FETCH_AND_OP_WORD (add, , +) 117 FETCH_AND_OP_WORD (sub, , -) 118 FETCH_AND_OP_WORD (or, , |) 119 FETCH_AND_OP_WORD (and, , &) 120 FETCH_AND_OP_WORD (xor, , ^) 121 FETCH_AND_OP_WORD (nand, ~, &) 122 123 #define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH 124 #define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH 125 126 /* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for 127 subword-sized quantities. */ 128 129 #define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN) \ 130 TYPE HIDDEN \ 131 NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val) \ 132 { \ 133 int *wordptr = (int *) ((unsigned int) ptr & ~3); \ 134 unsigned int mask, shift, oldval, newval; \ 135 int failure; \ 136 \ 137 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \ 138 mask = MASK_##WIDTH << shift; \ 139 \ 140 do { \ 141 oldval = *wordptr; \ 142 newval = ((PFX_OP (((oldval & mask) >> shift) \ 143 INF_OP (unsigned int) val)) << shift) & mask; \ 144 newval |= oldval & ~mask; \ 145 failure = __kernel_cmpxchg (oldval, newval, wordptr); \ 146 } while (failure != 0); \ 147 \ 148 return (RETURN & mask) >> shift; \ 149 } 150 151 SUBWORD_SYNC_OP (add, , +, short, 2, oldval) 152 SUBWORD_SYNC_OP (sub, , -, short, 2, oldval) 153 SUBWORD_SYNC_OP (or, , |, short, 2, oldval) 154 SUBWORD_SYNC_OP (and, , &, short, 2, oldval) 155 SUBWORD_SYNC_OP (xor, , ^, short, 2, oldval) 156 SUBWORD_SYNC_OP (nand, ~, &, short, 2, oldval) 157 158 SUBWORD_SYNC_OP (add, , +, signed char, 1, oldval) 159 SUBWORD_SYNC_OP (sub, , -, signed char, 1, oldval) 160 SUBWORD_SYNC_OP (or, , |, signed char, 1, oldval) 161 SUBWORD_SYNC_OP (and, , &, signed char, 1, oldval) 162 SUBWORD_SYNC_OP (xor, , ^, signed char, 1, oldval) 163 SUBWORD_SYNC_OP (nand, ~, &, signed char, 1, oldval) 164 165 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \ 166 int HIDDEN \ 167 __sync_##OP##_and_fetch_4 (int *ptr, int val) \ 168 { \ 169 int tmp, failure; \ 170 \ 171 do { \ 172 tmp = *ptr; \ 173 failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr); \ 174 } while (failure != 0); \ 175 \ 176 return PFX_OP (tmp INF_OP val); \ 177 } 178 179 OP_AND_FETCH_WORD (add, , +) 180 OP_AND_FETCH_WORD (sub, , -) 181 OP_AND_FETCH_WORD (or, , |) 182 OP_AND_FETCH_WORD (and, , &) 183 OP_AND_FETCH_WORD (xor, , ^) 184 OP_AND_FETCH_WORD (nand, ~, &) 185 186 SUBWORD_SYNC_OP (add, , +, short, 2, newval) 187 SUBWORD_SYNC_OP (sub, , -, short, 2, newval) 188 SUBWORD_SYNC_OP (or, , |, short, 2, newval) 189 SUBWORD_SYNC_OP (and, , &, short, 2, newval) 190 SUBWORD_SYNC_OP (xor, , ^, short, 2, newval) 191 SUBWORD_SYNC_OP (nand, ~, &, short, 2, newval) 192 193 SUBWORD_SYNC_OP (add, , +, signed char, 1, newval) 194 SUBWORD_SYNC_OP (sub, , -, signed char, 1, newval) 195 SUBWORD_SYNC_OP (or, , |, signed char, 1, newval) 196 SUBWORD_SYNC_OP (and, , &, signed char, 1, newval) 197 SUBWORD_SYNC_OP (xor, , ^, signed char, 1, newval) 198 SUBWORD_SYNC_OP (nand, ~, &, signed char, 1, newval) 199 200 int HIDDEN 201 __sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval) 202 { 203 int actual_oldval, fail; 204 205 while (1) 206 { 207 actual_oldval = *ptr; 208 209 if (__builtin_expect (oldval != actual_oldval, 0)) 210 return actual_oldval; 211 212 fail = __kernel_cmpxchg (actual_oldval, newval, ptr); 213 214 if (__builtin_expect (!fail, 1)) 215 return oldval; 216 } 217 } 218 219 #define SUBWORD_VAL_CAS(TYPE, WIDTH) \ 220 TYPE HIDDEN \ 221 __sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \ 222 TYPE newval) \ 223 { \ 224 int *wordptr = (int *)((unsigned int) ptr & ~3), fail; \ 225 unsigned int mask, shift, actual_oldval, actual_newval; \ 226 \ 227 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \ 228 mask = MASK_##WIDTH << shift; \ 229 \ 230 while (1) \ 231 { \ 232 actual_oldval = *wordptr; \ 233 \ 234 if (__builtin_expect (((actual_oldval & mask) >> shift) != \ 235 ((unsigned int) oldval & MASK_##WIDTH), 0)) \ 236 return (actual_oldval & mask) >> shift; \ 237 \ 238 actual_newval = (actual_oldval & ~mask) \ 239 | (((unsigned int) newval << shift) & mask); \ 240 \ 241 fail = __kernel_cmpxchg (actual_oldval, actual_newval, \ 242 wordptr); \ 243 \ 244 if (__builtin_expect (!fail, 1)) \ 245 return oldval; \ 246 } \ 247 } 248 249 SUBWORD_VAL_CAS (short, 2) 250 SUBWORD_VAL_CAS (signed char, 1) 251 252 typedef unsigned char bool; 253 254 bool HIDDEN 255 __sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval) 256 { 257 int failure = __kernel_cmpxchg (oldval, newval, ptr); 258 return (failure == 0); 259 } 260 261 #define SUBWORD_BOOL_CAS(TYPE, WIDTH) \ 262 bool HIDDEN \ 263 __sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \ 264 TYPE newval) \ 265 { \ 266 TYPE actual_oldval \ 267 = __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval); \ 268 return (oldval == actual_oldval); \ 269 } 270 271 SUBWORD_BOOL_CAS (short, 2) 272 SUBWORD_BOOL_CAS (signed char, 1) 273 274 void HIDDEN 275 __sync_synchronize (void) 276 { 277 __kernel_dmb (); 278 } 279 280 int HIDDEN 281 __sync_lock_test_and_set_4 (int *ptr, int val) 282 { 283 int failure, oldval; 284 285 do { 286 oldval = *ptr; 287 failure = __kernel_cmpxchg (oldval, val, ptr); 288 } while (failure != 0); 289 290 return oldval; 291 } 292 293 #define SUBWORD_TEST_AND_SET(TYPE, WIDTH) \ 294 TYPE HIDDEN \ 295 __sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val) \ 296 { \ 297 int failure; \ 298 unsigned int oldval, newval, shift, mask; \ 299 int *wordptr = (int *) ((unsigned int) ptr & ~3); \ 300 \ 301 shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \ 302 mask = MASK_##WIDTH << shift; \ 303 \ 304 do { \ 305 oldval = *wordptr; \ 306 newval = (oldval & ~mask) \ 307 | (((unsigned int) val << shift) & mask); \ 308 failure = __kernel_cmpxchg (oldval, newval, wordptr); \ 309 } while (failure != 0); \ 310 \ 311 return (oldval & mask) >> shift; \ 312 } 313 314 SUBWORD_TEST_AND_SET (short, 2) 315 SUBWORD_TEST_AND_SET (signed char, 1) 316 317 #define SYNC_LOCK_RELEASE(TYPE, WIDTH) \ 318 void HIDDEN \ 319 __sync_lock_release_##WIDTH (TYPE *ptr) \ 320 { \ 321 /* All writes before this point must be seen before we release \ 322 the lock itself. */ \ 323 __kernel_dmb (); \ 324 *ptr = 0; \ 325 } 326 327 SYNC_LOCK_RELEASE (long long, 8) 328 SYNC_LOCK_RELEASE (int, 4) 329 SYNC_LOCK_RELEASE (short, 2) 330 SYNC_LOCK_RELEASE (char, 1) 331