1 /* Linux-specific atomic operations for PA Linux. 2 Copyright (C) 2008-2017 Free Software Foundation, Inc. 3 Based on code contributed by CodeSourcery for ARM EABI Linux. 4 Modifications for PA Linux by Helge Deller <deller@gmx.de> 5 6 This file is part of GCC. 7 8 GCC is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free 10 Software Foundation; either version 3, or (at your option) any later 11 version. 12 13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 for more details. 17 18 Under Section 7 of GPL version 3, you are granted additional 19 permissions described in the GCC Runtime Library Exception, version 20 3.1, as published by the Free Software Foundation. 21 22 You should have received a copy of the GNU General Public License and 23 a copy of the GCC Runtime Library Exception along with this program; 24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 25 <http://www.gnu.org/licenses/>. */ 26 27 #define EFAULT 14 28 #define EBUSY 16 29 #define ENOSYS 251 30 31 /* PA-RISC 2.0 supports out-of-order execution for loads and stores. 32 Thus, we need to synchonize memory accesses. For more info, see: 33 "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt. 34 35 We implement byte, short and int versions of each atomic operation 36 using the kernel helper defined below. There is no support for 37 64-bit operations yet. */ 38 39 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace). */ 40 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1) 41 42 /* Kernel helper for compare-and-exchange a 32-bit value. */ 43 static inline long 44 __kernel_cmpxchg (int *mem, int oldval, int newval) 45 { 46 register unsigned long lws_mem asm("r26") = (unsigned long) (mem); 47 register int lws_old asm("r25") = oldval; 48 register int lws_new asm("r24") = newval; 49 register long lws_ret asm("r28"); 50 register long lws_errno asm("r21"); 51 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t" 52 "ldi %2, %%r20 \n\t" 53 : "=r" (lws_ret), "=r" (lws_errno) 54 : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new) 55 : "r1", "r20", "r22", "r23", "r29", "r31", "memory" 56 ); 57 if (__builtin_expect (lws_errno == -EFAULT || lws_errno == -ENOSYS, 0)) 58 __builtin_trap (); 59 60 /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains 61 the old value from memory. If this value is equal to OLDVAL, the 62 new value was written to memory. If not, return -EBUSY. */ 63 if (!lws_errno && lws_ret != oldval) 64 lws_errno = -EBUSY; 65 66 return lws_errno; 67 } 68 69 static inline long 70 __kernel_cmpxchg2 (void *mem, const void *oldval, const void *newval, 71 int val_size) 72 { 73 register unsigned long lws_mem asm("r26") = (unsigned long) (mem); 74 register unsigned long lws_old asm("r25") = (unsigned long) oldval; 75 register unsigned long lws_new asm("r24") = (unsigned long) newval; 76 register int lws_size asm("r23") = val_size; 77 register long lws_ret asm("r28"); 78 register long lws_errno asm("r21"); 79 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t" 80 "ldi %6, %%r20 \n\t" 81 : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem), 82 "+r" (lws_old), "+r" (lws_new), "+r" (lws_size) 83 : "i" (2) 84 : "r1", "r20", "r22", "r29", "r31", "fr4", "memory" 85 ); 86 87 /* If the kernel LWS call is successful, lws_ret contains 0. */ 88 if (__builtin_expect (lws_ret == 0, 1)) 89 return 0; 90 91 if (__builtin_expect (lws_errno == -EFAULT || lws_errno == -ENOSYS, 0)) 92 __builtin_trap (); 93 94 /* If the kernel LWS call fails with no error, return -EBUSY */ 95 if (__builtin_expect (!lws_errno, 0)) 96 return -EBUSY; 97 98 return lws_errno; 99 } 100 #define HIDDEN __attribute__ ((visibility ("hidden"))) 101 102 /* Big endian masks */ 103 #define INVERT_MASK_1 24 104 #define INVERT_MASK_2 16 105 106 #define MASK_1 0xffu 107 #define MASK_2 0xffffu 108 109 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \ 110 TYPE HIDDEN \ 111 __sync_fetch_and_##OP##_##WIDTH (TYPE *ptr, TYPE val) \ 112 { \ 113 TYPE tmp, newval; \ 114 long failure; \ 115 \ 116 do { \ 117 tmp = __atomic_load_n (ptr, __ATOMIC_RELAXED); \ 118 newval = PFX_OP (tmp INF_OP val); \ 119 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \ 120 } while (failure != 0); \ 121 \ 122 return tmp; \ 123 } 124 125 FETCH_AND_OP_2 (add, , +, long long, 8, 3) 126 FETCH_AND_OP_2 (sub, , -, long long, 8, 3) 127 FETCH_AND_OP_2 (or, , |, long long, 8, 3) 128 FETCH_AND_OP_2 (and, , &, long long, 8, 3) 129 FETCH_AND_OP_2 (xor, , ^, long long, 8, 3) 130 FETCH_AND_OP_2 (nand, ~, &, long long, 8, 3) 131 132 FETCH_AND_OP_2 (add, , +, short, 2, 1) 133 FETCH_AND_OP_2 (sub, , -, short, 2, 1) 134 FETCH_AND_OP_2 (or, , |, short, 2, 1) 135 FETCH_AND_OP_2 (and, , &, short, 2, 1) 136 FETCH_AND_OP_2 (xor, , ^, short, 2, 1) 137 FETCH_AND_OP_2 (nand, ~, &, short, 2, 1) 138 139 FETCH_AND_OP_2 (add, , +, signed char, 1, 0) 140 FETCH_AND_OP_2 (sub, , -, signed char, 1, 0) 141 FETCH_AND_OP_2 (or, , |, signed char, 1, 0) 142 FETCH_AND_OP_2 (and, , &, signed char, 1, 0) 143 FETCH_AND_OP_2 (xor, , ^, signed char, 1, 0) 144 FETCH_AND_OP_2 (nand, ~, &, signed char, 1, 0) 145 146 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \ 147 TYPE HIDDEN \ 148 __sync_##OP##_and_fetch_##WIDTH (TYPE *ptr, TYPE val) \ 149 { \ 150 TYPE tmp, newval; \ 151 long failure; \ 152 \ 153 do { \ 154 tmp = __atomic_load_n (ptr, __ATOMIC_RELAXED); \ 155 newval = PFX_OP (tmp INF_OP val); \ 156 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \ 157 } while (failure != 0); \ 158 \ 159 return PFX_OP (tmp INF_OP val); \ 160 } 161 162 OP_AND_FETCH_2 (add, , +, long long, 8, 3) 163 OP_AND_FETCH_2 (sub, , -, long long, 8, 3) 164 OP_AND_FETCH_2 (or, , |, long long, 8, 3) 165 OP_AND_FETCH_2 (and, , &, long long, 8, 3) 166 OP_AND_FETCH_2 (xor, , ^, long long, 8, 3) 167 OP_AND_FETCH_2 (nand, ~, &, long long, 8, 3) 168 169 OP_AND_FETCH_2 (add, , +, short, 2, 1) 170 OP_AND_FETCH_2 (sub, , -, short, 2, 1) 171 OP_AND_FETCH_2 (or, , |, short, 2, 1) 172 OP_AND_FETCH_2 (and, , &, short, 2, 1) 173 OP_AND_FETCH_2 (xor, , ^, short, 2, 1) 174 OP_AND_FETCH_2 (nand, ~, &, short, 2, 1) 175 176 OP_AND_FETCH_2 (add, , +, signed char, 1, 0) 177 OP_AND_FETCH_2 (sub, , -, signed char, 1, 0) 178 OP_AND_FETCH_2 (or, , |, signed char, 1, 0) 179 OP_AND_FETCH_2 (and, , &, signed char, 1, 0) 180 OP_AND_FETCH_2 (xor, , ^, signed char, 1, 0) 181 OP_AND_FETCH_2 (nand, ~, &, signed char, 1, 0) 182 183 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \ 184 int HIDDEN \ 185 __sync_fetch_and_##OP##_4 (int *ptr, int val) \ 186 { \ 187 int tmp; \ 188 long failure; \ 189 \ 190 do { \ 191 tmp = __atomic_load_n (ptr, __ATOMIC_RELAXED); \ 192 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \ 193 } while (failure != 0); \ 194 \ 195 return tmp; \ 196 } 197 198 FETCH_AND_OP_WORD (add, , +) 199 FETCH_AND_OP_WORD (sub, , -) 200 FETCH_AND_OP_WORD (or, , |) 201 FETCH_AND_OP_WORD (and, , &) 202 FETCH_AND_OP_WORD (xor, , ^) 203 FETCH_AND_OP_WORD (nand, ~, &) 204 205 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \ 206 int HIDDEN \ 207 __sync_##OP##_and_fetch_4 (int *ptr, int val) \ 208 { \ 209 int tmp; \ 210 long failure; \ 211 \ 212 do { \ 213 tmp = __atomic_load_n (ptr, __ATOMIC_RELAXED); \ 214 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \ 215 } while (failure != 0); \ 216 \ 217 return PFX_OP (tmp INF_OP val); \ 218 } 219 220 OP_AND_FETCH_WORD (add, , +) 221 OP_AND_FETCH_WORD (sub, , -) 222 OP_AND_FETCH_WORD (or, , |) 223 OP_AND_FETCH_WORD (and, , &) 224 OP_AND_FETCH_WORD (xor, , ^) 225 OP_AND_FETCH_WORD (nand, ~, &) 226 227 typedef unsigned char bool; 228 229 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX) \ 230 TYPE HIDDEN \ 231 __sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \ 232 TYPE newval) \ 233 { \ 234 TYPE actual_oldval; \ 235 long fail; \ 236 \ 237 while (1) \ 238 { \ 239 actual_oldval = __atomic_load_n (ptr, __ATOMIC_RELAXED); \ 240 \ 241 if (__builtin_expect (oldval != actual_oldval, 0)) \ 242 return actual_oldval; \ 243 \ 244 fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \ 245 \ 246 if (__builtin_expect (!fail, 1)) \ 247 return actual_oldval; \ 248 } \ 249 } \ 250 \ 251 bool HIDDEN \ 252 __sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \ 253 TYPE newval) \ 254 { \ 255 long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX); \ 256 return (failure == 0); \ 257 } 258 259 COMPARE_AND_SWAP_2 (long long, 8, 3) 260 COMPARE_AND_SWAP_2 (short, 2, 1) 261 COMPARE_AND_SWAP_2 (char, 1, 0) 262 263 int HIDDEN 264 __sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval) 265 { 266 long fail; 267 int actual_oldval; 268 269 while (1) 270 { 271 actual_oldval = __atomic_load_n (ptr, __ATOMIC_RELAXED); 272 273 if (__builtin_expect (oldval != actual_oldval, 0)) 274 return actual_oldval; 275 276 fail = __kernel_cmpxchg (ptr, actual_oldval, newval); 277 278 if (__builtin_expect (!fail, 1)) 279 return actual_oldval; 280 } 281 } 282 283 bool HIDDEN 284 __sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval) 285 { 286 long failure = __kernel_cmpxchg (ptr, oldval, newval); 287 return (failure == 0); 288 } 289 290 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX) \ 291 TYPE HIDDEN \ 292 __sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val) \ 293 { \ 294 TYPE oldval; \ 295 long failure; \ 296 \ 297 do { \ 298 oldval = __atomic_load_n (ptr, __ATOMIC_RELAXED); \ 299 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \ 300 } while (failure != 0); \ 301 \ 302 return oldval; \ 303 } 304 305 SYNC_LOCK_TEST_AND_SET_2 (long long, 8, 3) 306 SYNC_LOCK_TEST_AND_SET_2 (short, 2, 1) 307 SYNC_LOCK_TEST_AND_SET_2 (signed char, 1, 0) 308 309 int HIDDEN 310 __sync_lock_test_and_set_4 (int *ptr, int val) 311 { 312 long failure; 313 int oldval; 314 315 do { 316 oldval = __atomic_load_n (ptr, __ATOMIC_RELAXED); 317 failure = __kernel_cmpxchg (ptr, oldval, val); 318 } while (failure != 0); 319 320 return oldval; 321 } 322 323 void HIDDEN 324 __sync_lock_release_8 (long long *ptr) 325 { 326 /* All accesses must be complete before we release the lock. */ 327 __sync_synchronize (); 328 *(double *)ptr = 0; 329 } 330 331 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH) \ 332 void HIDDEN \ 333 __sync_lock_release_##WIDTH (TYPE *ptr) \ 334 { \ 335 /* All accesses must be complete before we release \ 336 the lock. */ \ 337 __sync_synchronize (); \ 338 *ptr = 0; \ 339 } 340 341 SYNC_LOCK_RELEASE_1 (int, 4) 342 SYNC_LOCK_RELEASE_1 (short, 2) 343 SYNC_LOCK_RELEASE_1 (signed char, 1) 344