1 /* TILE atomics. 2 Copyright (C) 2011-2013 Free Software Foundation, Inc. 3 Contributed by Walter Lee (walt@tilera.com) 4 5 This file is free software; you can redistribute it and/or modify it 6 under the terms of the GNU General Public License as published by the 7 Free Software Foundation; either version 3, or (at your option) any 8 later version. 9 10 This file is distributed in the hope that it will be useful, but 11 WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 General Public License for more details. 14 15 Under Section 7 of GPL version 3, you are granted additional 16 permissions described in the GCC Runtime Library Exception, version 17 3.1, as published by the Free Software Foundation. 18 19 You should have received a copy of the GNU General Public License and 20 a copy of the GCC Runtime Library Exception along with this program; 21 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 22 <http://www.gnu.org/licenses/>. */ 23 24 #include "system.h" 25 #include "coretypes.h" 26 #include "atomic.h" 27 28 /* This code should be inlined by the compiler, but for now support 29 it as out-of-line methods in libgcc. */ 30 31 static void 32 pre_atomic_barrier (int model) 33 { 34 switch ((enum memmodel) model) 35 { 36 case MEMMODEL_RELEASE: 37 case MEMMODEL_ACQ_REL: 38 case MEMMODEL_SEQ_CST: 39 __atomic_thread_fence (model); 40 break; 41 default: 42 break; 43 } 44 return; 45 } 46 47 static void 48 post_atomic_barrier (int model) 49 { 50 switch ((enum memmodel) model) 51 { 52 case MEMMODEL_ACQUIRE: 53 case MEMMODEL_ACQ_REL: 54 case MEMMODEL_SEQ_CST: 55 __atomic_thread_fence (model); 56 break; 57 default: 58 break; 59 } 60 return; 61 } 62 63 #define __unused __attribute__((unused)) 64 65 #define __atomic_fetch_and_do(type, size, opname) \ 66 type \ 67 __atomic_fetch_##opname##_##size(type* p, type i, int model) \ 68 { \ 69 pre_atomic_barrier(model); \ 70 type rv = arch_atomic_##opname(p, i); \ 71 post_atomic_barrier(model); \ 72 return rv; \ 73 } 74 75 __atomic_fetch_and_do (int, 4, add) 76 __atomic_fetch_and_do (int, 4, sub) 77 __atomic_fetch_and_do (int, 4, or) 78 __atomic_fetch_and_do (int, 4, and) 79 __atomic_fetch_and_do (int, 4, xor) 80 __atomic_fetch_and_do (int, 4, nand) 81 __atomic_fetch_and_do (long long, 8, add) 82 __atomic_fetch_and_do (long long, 8, sub) 83 __atomic_fetch_and_do (long long, 8, or) 84 __atomic_fetch_and_do (long long, 8, and) 85 __atomic_fetch_and_do (long long, 8, xor) 86 __atomic_fetch_and_do (long long, 8, nand) 87 #define __atomic_do_and_fetch(type, size, opname, op) \ 88 type \ 89 __atomic_##opname##_fetch_##size(type* p, type i, int model) \ 90 { \ 91 pre_atomic_barrier(model); \ 92 type rv = arch_atomic_##opname(p, i) op i; \ 93 post_atomic_barrier(model); \ 94 return rv; \ 95 } 96 __atomic_do_and_fetch (int, 4, add, +) 97 __atomic_do_and_fetch (int, 4, sub, -) 98 __atomic_do_and_fetch (int, 4, or, |) 99 __atomic_do_and_fetch (int, 4, and, &) 100 __atomic_do_and_fetch (int, 4, xor, |) 101 __atomic_do_and_fetch (int, 4, nand, &) 102 __atomic_do_and_fetch (long long, 8, add, +) 103 __atomic_do_and_fetch (long long, 8, sub, -) 104 __atomic_do_and_fetch (long long, 8, or, |) 105 __atomic_do_and_fetch (long long, 8, and, &) 106 __atomic_do_and_fetch (long long, 8, xor, |) 107 __atomic_do_and_fetch (long long, 8, nand, &) 108 #define __atomic_exchange_methods(type, size) \ 109 bool \ 110 __atomic_compare_exchange_##size(volatile type* ptr, type* oldvalp, \ 111 type newval, bool weak __unused, \ 112 int models, int modelf __unused) \ 113 { \ 114 type oldval = *oldvalp; \ 115 pre_atomic_barrier(models); \ 116 type retval = arch_atomic_val_compare_and_exchange(ptr, oldval, newval); \ 117 post_atomic_barrier(models); \ 118 bool success = (retval == oldval); \ 119 *oldvalp = retval; \ 120 return success; \ 121 } \ 122 \ 123 type \ 124 __atomic_exchange_##size(volatile type* ptr, type val, int model) \ 125 { \ 126 pre_atomic_barrier(model); \ 127 type retval = arch_atomic_exchange(ptr, val); \ 128 post_atomic_barrier(model); \ 129 return retval; \ 130 } 131 __atomic_exchange_methods (int, 4) 132 __atomic_exchange_methods (long long, 8) 133 134 /* Subword methods require the same approach for both TILEPro and 135 TILE-Gx. We load the background data for the word, insert the 136 desired subword piece, then compare-and-exchange it into place. */ 137 #define u8 unsigned char 138 #define u16 unsigned short 139 #define __atomic_subword_cmpxchg(type, size) \ 140 \ 141 bool \ 142 __atomic_compare_exchange_##size(volatile type* ptr, type* guess, \ 143 type val, bool weak __unused, int models, \ 144 int modelf __unused) \ 145 { \ 146 pre_atomic_barrier(models); \ 147 unsigned int *p = (unsigned int *)((unsigned long)ptr & ~3UL); \ 148 const int shift = ((unsigned long)ptr & 3UL) * 8; \ 149 const unsigned int valmask = (1 << (sizeof(type) * 8)) - 1; \ 150 const unsigned int bgmask = ~(valmask << shift); \ 151 unsigned int oldword = *p; \ 152 type oldval = (oldword >> shift) & valmask; \ 153 if (__builtin_expect((oldval == *guess), 1)) { \ 154 unsigned int word = (oldword & bgmask) | ((val & valmask) << shift); \ 155 oldword = arch_atomic_val_compare_and_exchange(p, oldword, word); \ 156 oldval = (oldword >> shift) & valmask; \ 157 } \ 158 post_atomic_barrier(models); \ 159 bool success = (oldval == *guess); \ 160 *guess = oldval; \ 161 return success; \ 162 } 163 __atomic_subword_cmpxchg (u8, 1) 164 __atomic_subword_cmpxchg (u16, 2) 165 /* For the atomic-update subword methods, we use the same approach as 166 above, but we retry until we succeed if the compare-and-exchange 167 fails. */ 168 #define __atomic_subword(type, proto, top, expr, bottom) \ 169 proto \ 170 { \ 171 top \ 172 unsigned int *p = (unsigned int *)((unsigned long)ptr & ~3UL); \ 173 const int shift = ((unsigned long)ptr & 3UL) * 8; \ 174 const unsigned int valmask = (1 << (sizeof(type) * 8)) - 1; \ 175 const unsigned int bgmask = ~(valmask << shift); \ 176 unsigned int oldword, xword = *p; \ 177 type val, oldval; \ 178 do { \ 179 oldword = xword; \ 180 oldval = (oldword >> shift) & valmask; \ 181 val = expr; \ 182 unsigned int word = (oldword & bgmask) | ((val & valmask) << shift); \ 183 xword = arch_atomic_val_compare_and_exchange(p, oldword, word); \ 184 } while (__builtin_expect(xword != oldword, 0)); \ 185 bottom \ 186 } 187 #define __atomic_subword_fetch(type, funcname, expr, retval) \ 188 __atomic_subword(type, \ 189 type __atomic_ ## funcname(volatile type *ptr, type i, int model), \ 190 pre_atomic_barrier(model);, \ 191 expr, \ 192 post_atomic_barrier(model); return retval;) 193 __atomic_subword_fetch (u8, fetch_add_1, oldval + i, oldval) 194 __atomic_subword_fetch (u8, fetch_sub_1, oldval - i, oldval) 195 __atomic_subword_fetch (u8, fetch_or_1, oldval | i, oldval) 196 __atomic_subword_fetch (u8, fetch_and_1, oldval & i, oldval) 197 __atomic_subword_fetch (u8, fetch_xor_1, oldval ^ i, oldval) 198 __atomic_subword_fetch (u8, fetch_nand_1, ~(oldval & i), oldval) 199 __atomic_subword_fetch (u16, fetch_add_2, oldval + i, oldval) 200 __atomic_subword_fetch (u16, fetch_sub_2, oldval - i, oldval) 201 __atomic_subword_fetch (u16, fetch_or_2, oldval | i, oldval) 202 __atomic_subword_fetch (u16, fetch_and_2, oldval & i, oldval) 203 __atomic_subword_fetch (u16, fetch_xor_2, oldval ^ i, oldval) 204 __atomic_subword_fetch (u16, fetch_nand_2, ~(oldval & i), oldval) 205 __atomic_subword_fetch (u8, add_fetch_1, oldval + i, val) 206 __atomic_subword_fetch (u8, sub_fetch_1, oldval - i, val) 207 __atomic_subword_fetch (u8, or_fetch_1, oldval | i, val) 208 __atomic_subword_fetch (u8, and_fetch_1, oldval & i, val) 209 __atomic_subword_fetch (u8, xor_fetch_1, oldval ^ i, val) 210 __atomic_subword_fetch (u8, nand_fetch_1, ~(oldval & i), val) 211 __atomic_subword_fetch (u16, add_fetch_2, oldval + i, val) 212 __atomic_subword_fetch (u16, sub_fetch_2, oldval - i, val) 213 __atomic_subword_fetch (u16, or_fetch_2, oldval | i, val) 214 __atomic_subword_fetch (u16, and_fetch_2, oldval & i, val) 215 __atomic_subword_fetch (u16, xor_fetch_2, oldval ^ i, val) 216 __atomic_subword_fetch (u16, nand_fetch_2, ~(oldval & i), val) 217 #define __atomic_subword_lock(type, size) \ 218 \ 219 __atomic_subword(type, \ 220 type __atomic_exchange_##size(volatile type* ptr, type nval, int model), \ 221 pre_atomic_barrier(model);, \ 222 nval, \ 223 post_atomic_barrier(model); return oldval;) 224 __atomic_subword_lock (u8, 1) 225 __atomic_subword_lock (u16, 2) 226