1 /* Linux-specific atomic operations for PA Linux.
2 Copyright (C) 2008-2020 Free Software Foundation, Inc.
3 Based on code contributed by CodeSourcery for ARM EABI Linux.
4 Modifications for PA Linux by Helge Deller <deller@gmx.de>
5
6 This file is part of GCC.
7
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
12
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
17
18 Under Section 7 of GPL version 3, you are granted additional
19 permissions described in the GCC Runtime Library Exception, version
20 3.1, as published by the Free Software Foundation.
21
22 You should have received a copy of the GNU General Public License and
23 a copy of the GCC Runtime Library Exception along with this program;
24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
25 <http://www.gnu.org/licenses/>. */
26
27 #define EFAULT 14
28 #define EBUSY 16
29 #define ENOSYS 251
30
31 typedef unsigned char u8;
32 typedef short unsigned int u16;
33 #ifdef __LP64__
34 typedef long unsigned int u64;
35 #else
36 typedef long long unsigned int u64;
37 #endif
38
39 /* PA-RISC 2.0 supports out-of-order execution for loads and stores.
40 Thus, we need to synchonize memory accesses. For more info, see:
41 "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt.
42
43 We implement byte, short and int versions of each atomic operation
44 using the kernel helper defined below. There is no support for
45 64-bit operations yet. */
46
47 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace). */
48 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1)
49
50 /* Kernel helper for compare-and-exchange a 32-bit value. */
51 static inline long
__kernel_cmpxchg(volatile void * mem,int oldval,int newval)52 __kernel_cmpxchg (volatile void *mem, int oldval, int newval)
53 {
54 register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
55 register int lws_old asm("r25") = oldval;
56 register int lws_new asm("r24") = newval;
57 register long lws_ret asm("r28");
58 register long lws_errno asm("r21");
59 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t"
60 "ldi %2, %%r20 \n\t"
61 : "=r" (lws_ret), "=r" (lws_errno)
62 : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new)
63 : "r1", "r20", "r22", "r23", "r29", "r31", "memory"
64 );
65
66 /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains
67 the old value from memory. If this value is equal to OLDVAL, the
68 new value was written to memory. If not, return -EBUSY. */
69 if (!lws_errno && lws_ret != oldval)
70 return -EBUSY;
71
72 return lws_errno;
73 }
74
75 static inline long
__kernel_cmpxchg2(volatile void * mem,const void * oldval,const void * newval,int val_size)76 __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval,
77 int val_size)
78 {
79 register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
80 register unsigned long lws_old asm("r25") = (unsigned long) oldval;
81 register unsigned long lws_new asm("r24") = (unsigned long) newval;
82 register int lws_size asm("r23") = val_size;
83 register long lws_ret asm("r28");
84 register long lws_errno asm("r21");
85 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t"
86 "ldi %6, %%r20 \n\t"
87 : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem),
88 "+r" (lws_old), "+r" (lws_new), "+r" (lws_size)
89 : "i" (2)
90 : "r1", "r20", "r22", "r29", "r31", "fr4", "memory"
91 );
92
93 /* If the kernel LWS call is successful, lws_ret contains 0. */
94 if (__builtin_expect (lws_ret == 0, 1))
95 return 0;
96
97 /* If the kernel LWS call fails with no error, return -EBUSY */
98 if (__builtin_expect (!lws_errno, 0))
99 return -EBUSY;
100
101 return lws_errno;
102 }
103 #define HIDDEN __attribute__ ((visibility ("hidden")))
104
105 /* Big endian masks */
106 #define INVERT_MASK_1 24
107 #define INVERT_MASK_2 16
108
109 #define MASK_1 0xffu
110 #define MASK_2 0xffffu
111
112 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \
113 TYPE HIDDEN \
114 __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val) \
115 { \
116 TYPE tmp, newval; \
117 long failure; \
118 \
119 do { \
120 tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED); \
121 newval = PFX_OP (tmp INF_OP val); \
122 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \
123 } while (failure != 0); \
124 \
125 return tmp; \
126 }
127
128 FETCH_AND_OP_2 (add, , +, u64, 8, 3)
129 FETCH_AND_OP_2 (sub, , -, u64, 8, 3)
130 FETCH_AND_OP_2 (or, , |, u64, 8, 3)
131 FETCH_AND_OP_2 (and, , &, u64, 8, 3)
132 FETCH_AND_OP_2 (xor, , ^, u64, 8, 3)
133 FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3)
134
135 FETCH_AND_OP_2 (add, , +, u16, 2, 1)
136 FETCH_AND_OP_2 (sub, , -, u16, 2, 1)
137 FETCH_AND_OP_2 (or, , |, u16, 2, 1)
138 FETCH_AND_OP_2 (and, , &, u16, 2, 1)
139 FETCH_AND_OP_2 (xor, , ^, u16, 2, 1)
140 FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1)
141
142 FETCH_AND_OP_2 (add, , +, u8, 1, 0)
143 FETCH_AND_OP_2 (sub, , -, u8, 1, 0)
144 FETCH_AND_OP_2 (or, , |, u8, 1, 0)
145 FETCH_AND_OP_2 (and, , &, u8, 1, 0)
146 FETCH_AND_OP_2 (xor, , ^, u8, 1, 0)
147 FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0)
148
149 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \
150 TYPE HIDDEN \
151 __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val) \
152 { \
153 TYPE tmp, newval; \
154 long failure; \
155 \
156 do { \
157 tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED); \
158 newval = PFX_OP (tmp INF_OP val); \
159 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \
160 } while (failure != 0); \
161 \
162 return PFX_OP (tmp INF_OP val); \
163 }
164
165 OP_AND_FETCH_2 (add, , +, u64, 8, 3)
166 OP_AND_FETCH_2 (sub, , -, u64, 8, 3)
167 OP_AND_FETCH_2 (or, , |, u64, 8, 3)
168 OP_AND_FETCH_2 (and, , &, u64, 8, 3)
169 OP_AND_FETCH_2 (xor, , ^, u64, 8, 3)
170 OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3)
171
172 OP_AND_FETCH_2 (add, , +, u16, 2, 1)
173 OP_AND_FETCH_2 (sub, , -, u16, 2, 1)
174 OP_AND_FETCH_2 (or, , |, u16, 2, 1)
175 OP_AND_FETCH_2 (and, , &, u16, 2, 1)
176 OP_AND_FETCH_2 (xor, , ^, u16, 2, 1)
177 OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1)
178
179 OP_AND_FETCH_2 (add, , +, u8, 1, 0)
180 OP_AND_FETCH_2 (sub, , -, u8, 1, 0)
181 OP_AND_FETCH_2 (or, , |, u8, 1, 0)
182 OP_AND_FETCH_2 (and, , &, u8, 1, 0)
183 OP_AND_FETCH_2 (xor, , ^, u8, 1, 0)
184 OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0)
185
186 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \
187 unsigned int HIDDEN \
188 __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val) \
189 { \
190 unsigned int tmp; \
191 long failure; \
192 \
193 do { \
194 tmp = __atomic_load_n ((volatile unsigned int *)ptr, \
195 __ATOMIC_RELAXED); \
196 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \
197 } while (failure != 0); \
198 \
199 return tmp; \
200 }
201
202 FETCH_AND_OP_WORD (add, , +)
203 FETCH_AND_OP_WORD (sub, , -)
204 FETCH_AND_OP_WORD (or, , |)
205 FETCH_AND_OP_WORD (and, , &)
206 FETCH_AND_OP_WORD (xor, , ^)
207 FETCH_AND_OP_WORD (nand, ~, &)
208
209 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \
210 unsigned int HIDDEN \
211 __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val) \
212 { \
213 unsigned int tmp; \
214 long failure; \
215 \
216 do { \
217 tmp = __atomic_load_n ((volatile unsigned int *)ptr, \
218 __ATOMIC_RELAXED); \
219 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \
220 } while (failure != 0); \
221 \
222 return PFX_OP (tmp INF_OP val); \
223 }
224
225 OP_AND_FETCH_WORD (add, , +)
226 OP_AND_FETCH_WORD (sub, , -)
227 OP_AND_FETCH_WORD (or, , |)
228 OP_AND_FETCH_WORD (and, , &)
229 OP_AND_FETCH_WORD (xor, , ^)
230 OP_AND_FETCH_WORD (nand, ~, &)
231
232 typedef unsigned char bool;
233
234 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX) \
235 TYPE HIDDEN \
236 __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \
237 TYPE newval) \
238 { \
239 TYPE actual_oldval; \
240 long fail; \
241 \
242 while (1) \
243 { \
244 actual_oldval = __atomic_load_n ((volatile TYPE *)ptr, \
245 __ATOMIC_RELAXED); \
246 \
247 if (__builtin_expect (oldval != actual_oldval, 0)) \
248 return actual_oldval; \
249 \
250 fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \
251 \
252 if (__builtin_expect (!fail, 1)) \
253 return actual_oldval; \
254 } \
255 } \
256 \
257 _Bool HIDDEN \
258 __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr, \
259 TYPE oldval, TYPE newval) \
260 { \
261 long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX); \
262 return (failure == 0); \
263 }
264
265 COMPARE_AND_SWAP_2 (u64, 8, 3)
266 COMPARE_AND_SWAP_2 (u16, 2, 1)
267 COMPARE_AND_SWAP_2 (u8, 1, 0)
268
269 unsigned int HIDDEN
__sync_val_compare_and_swap_4(volatile void * ptr,unsigned int oldval,unsigned int newval)270 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
271 unsigned int newval)
272 {
273 long fail;
274 unsigned int actual_oldval;
275
276 while (1)
277 {
278 actual_oldval = __atomic_load_n ((volatile unsigned int *)ptr,
279 __ATOMIC_RELAXED);
280
281 if (__builtin_expect (oldval != actual_oldval, 0))
282 return actual_oldval;
283
284 fail = __kernel_cmpxchg (ptr, actual_oldval, newval);
285
286 if (__builtin_expect (!fail, 1))
287 return actual_oldval;
288 }
289 }
290
291 _Bool HIDDEN
__sync_bool_compare_and_swap_4(volatile void * ptr,unsigned int oldval,unsigned int newval)292 __sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
293 unsigned int newval)
294 {
295 long failure = __kernel_cmpxchg (ptr, oldval, newval);
296 return (failure == 0);
297 }
298
299 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX) \
300 TYPE HIDDEN \
301 __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val) \
302 { \
303 TYPE oldval; \
304 long failure; \
305 \
306 do { \
307 oldval = __atomic_load_n ((volatile TYPE *)ptr, \
308 __ATOMIC_RELAXED); \
309 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \
310 } while (failure != 0); \
311 \
312 return oldval; \
313 }
314
315 SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3)
316 SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1)
317 SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0)
318
319 unsigned int HIDDEN
__sync_lock_test_and_set_4(volatile void * ptr,unsigned int val)320 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
321 {
322 long failure;
323 unsigned int oldval;
324
325 do {
326 oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED);
327 failure = __kernel_cmpxchg (ptr, oldval, val);
328 } while (failure != 0);
329
330 return oldval;
331 }
332
333 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX) \
334 void HIDDEN \
335 __sync_lock_release_##WIDTH (volatile void *ptr) \
336 { \
337 TYPE oldval, val = 0; \
338 long failure; \
339 \
340 do { \
341 oldval = __atomic_load_n ((volatile TYPE *)ptr, \
342 __ATOMIC_RELAXED); \
343 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \
344 } while (failure != 0); \
345 }
346
347 SYNC_LOCK_RELEASE_1 (u64, 8, 3)
348 SYNC_LOCK_RELEASE_1 (u16, 2, 1)
349 SYNC_LOCK_RELEASE_1 (u8, 1, 0)
350
351 void HIDDEN
__sync_lock_release_4(volatile void * ptr)352 __sync_lock_release_4 (volatile void *ptr)
353 {
354 long failure;
355 unsigned int oldval;
356
357 do {
358 oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED);
359 failure = __kernel_cmpxchg (ptr, oldval, 0);
360 } while (failure != 0);
361 }
362