1 // Internal macros for the simd implementation -*- C++ -*- 2 3 // Copyright (C) 2020-2022 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_DETAIL_H_ 26 #define _GLIBCXX_EXPERIMENTAL_SIMD_DETAIL_H_ 27 28 #if __cplusplus >= 201703L 29 30 #include <cstddef> 31 #include <cstdint> 32 33 /// @cond undocumented 34 35 #define _GLIBCXX_SIMD_BEGIN_NAMESPACE \ 36 namespace std _GLIBCXX_VISIBILITY(default) \ 37 { \ 38 _GLIBCXX_BEGIN_NAMESPACE_VERSION \ 39 namespace experimental { \ 40 inline namespace parallelism_v2 { 41 #define _GLIBCXX_SIMD_END_NAMESPACE \ 42 } \ 43 } \ 44 _GLIBCXX_END_NAMESPACE_VERSION \ 45 } 46 47 // ISA extension detection. The following defines all the _GLIBCXX_SIMD_HAVE_XXX 48 // macros ARM{{{ 49 #if defined __ARM_NEON 50 #define _GLIBCXX_SIMD_HAVE_NEON 1 51 #else 52 #define _GLIBCXX_SIMD_HAVE_NEON 0 53 #endif 54 #if defined __ARM_NEON && (__ARM_ARCH >= 8 || defined __aarch64__) 55 #define _GLIBCXX_SIMD_HAVE_NEON_A32 1 56 #else 57 #define _GLIBCXX_SIMD_HAVE_NEON_A32 0 58 #endif 59 #if defined __ARM_NEON && defined __aarch64__ 60 #define _GLIBCXX_SIMD_HAVE_NEON_A64 1 61 #else 62 #define _GLIBCXX_SIMD_HAVE_NEON_A64 0 63 #endif 64 //}}} 65 // x86{{{ 66 #ifdef __MMX__ 67 #define _GLIBCXX_SIMD_HAVE_MMX 1 68 #else 69 #define _GLIBCXX_SIMD_HAVE_MMX 0 70 #endif 71 #if defined __SSE__ || defined __x86_64__ 72 #define _GLIBCXX_SIMD_HAVE_SSE 1 73 #else 74 #define _GLIBCXX_SIMD_HAVE_SSE 0 75 #endif 76 #if defined __SSE2__ || defined __x86_64__ 77 #define _GLIBCXX_SIMD_HAVE_SSE2 1 78 #else 79 #define _GLIBCXX_SIMD_HAVE_SSE2 0 80 #endif 81 #ifdef __SSE3__ 82 #define _GLIBCXX_SIMD_HAVE_SSE3 1 83 #else 84 #define _GLIBCXX_SIMD_HAVE_SSE3 0 85 #endif 86 #ifdef __SSSE3__ 87 #define _GLIBCXX_SIMD_HAVE_SSSE3 1 88 #else 89 #define _GLIBCXX_SIMD_HAVE_SSSE3 0 90 #endif 91 #ifdef __SSE4_1__ 92 #define _GLIBCXX_SIMD_HAVE_SSE4_1 1 93 #else 94 #define _GLIBCXX_SIMD_HAVE_SSE4_1 0 95 #endif 96 #ifdef __SSE4_2__ 97 #define _GLIBCXX_SIMD_HAVE_SSE4_2 1 98 #else 99 #define _GLIBCXX_SIMD_HAVE_SSE4_2 0 100 #endif 101 #ifdef __XOP__ 102 #define _GLIBCXX_SIMD_HAVE_XOP 1 103 #else 104 #define _GLIBCXX_SIMD_HAVE_XOP 0 105 #endif 106 #ifdef __AVX__ 107 #define _GLIBCXX_SIMD_HAVE_AVX 1 108 #else 109 #define _GLIBCXX_SIMD_HAVE_AVX 0 110 #endif 111 #ifdef __AVX2__ 112 #define _GLIBCXX_SIMD_HAVE_AVX2 1 113 #else 114 #define _GLIBCXX_SIMD_HAVE_AVX2 0 115 #endif 116 #ifdef __BMI__ 117 #define _GLIBCXX_SIMD_HAVE_BMI1 1 118 #else 119 #define _GLIBCXX_SIMD_HAVE_BMI1 0 120 #endif 121 #ifdef __BMI2__ 122 #define _GLIBCXX_SIMD_HAVE_BMI2 1 123 #else 124 #define _GLIBCXX_SIMD_HAVE_BMI2 0 125 #endif 126 #ifdef __LZCNT__ 127 #define _GLIBCXX_SIMD_HAVE_LZCNT 1 128 #else 129 #define _GLIBCXX_SIMD_HAVE_LZCNT 0 130 #endif 131 #ifdef __SSE4A__ 132 #define _GLIBCXX_SIMD_HAVE_SSE4A 1 133 #else 134 #define _GLIBCXX_SIMD_HAVE_SSE4A 0 135 #endif 136 #ifdef __FMA__ 137 #define _GLIBCXX_SIMD_HAVE_FMA 1 138 #else 139 #define _GLIBCXX_SIMD_HAVE_FMA 0 140 #endif 141 #ifdef __FMA4__ 142 #define _GLIBCXX_SIMD_HAVE_FMA4 1 143 #else 144 #define _GLIBCXX_SIMD_HAVE_FMA4 0 145 #endif 146 #ifdef __F16C__ 147 #define _GLIBCXX_SIMD_HAVE_F16C 1 148 #else 149 #define _GLIBCXX_SIMD_HAVE_F16C 0 150 #endif 151 #ifdef __POPCNT__ 152 #define _GLIBCXX_SIMD_HAVE_POPCNT 1 153 #else 154 #define _GLIBCXX_SIMD_HAVE_POPCNT 0 155 #endif 156 #ifdef __AVX512F__ 157 #define _GLIBCXX_SIMD_HAVE_AVX512F 1 158 #else 159 #define _GLIBCXX_SIMD_HAVE_AVX512F 0 160 #endif 161 #ifdef __AVX512DQ__ 162 #define _GLIBCXX_SIMD_HAVE_AVX512DQ 1 163 #else 164 #define _GLIBCXX_SIMD_HAVE_AVX512DQ 0 165 #endif 166 #ifdef __AVX512VL__ 167 #define _GLIBCXX_SIMD_HAVE_AVX512VL 1 168 #else 169 #define _GLIBCXX_SIMD_HAVE_AVX512VL 0 170 #endif 171 #ifdef __AVX512BW__ 172 #define _GLIBCXX_SIMD_HAVE_AVX512BW 1 173 #else 174 #define _GLIBCXX_SIMD_HAVE_AVX512BW 0 175 #endif 176 #ifdef __AVX512BITALG__ 177 #define _GLIBCXX_SIMD_HAVE_AVX512BITALG 1 178 #else 179 #define _GLIBCXX_SIMD_HAVE_AVX512BITALG 0 180 #endif 181 #ifdef __AVX512VBMI2__ 182 #define _GLIBCXX_SIMD_HAVE_AVX512VBMI2 1 183 #else 184 #define _GLIBCXX_SIMD_HAVE_AVX512VBMI2 0 185 #endif 186 #ifdef __AVX512VBMI__ 187 #define _GLIBCXX_SIMD_HAVE_AVX512VBMI 1 188 #else 189 #define _GLIBCXX_SIMD_HAVE_AVX512VBMI 0 190 #endif 191 #ifdef __AVX512IFMA__ 192 #define _GLIBCXX_SIMD_HAVE_AVX512IFMA 1 193 #else 194 #define _GLIBCXX_SIMD_HAVE_AVX512IFMA 0 195 #endif 196 #ifdef __AVX512CD__ 197 #define _GLIBCXX_SIMD_HAVE_AVX512CD 1 198 #else 199 #define _GLIBCXX_SIMD_HAVE_AVX512CD 0 200 #endif 201 #ifdef __AVX512VNNI__ 202 #define _GLIBCXX_SIMD_HAVE_AVX512VNNI 1 203 #else 204 #define _GLIBCXX_SIMD_HAVE_AVX512VNNI 0 205 #endif 206 #ifdef __AVX512VPOPCNTDQ__ 207 #define _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ 1 208 #else 209 #define _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ 0 210 #endif 211 #ifdef __AVX512VP2INTERSECT__ 212 #define _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT 1 213 #else 214 #define _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT 0 215 #endif 216 217 #if _GLIBCXX_SIMD_HAVE_SSE 218 #define _GLIBCXX_SIMD_HAVE_SSE_ABI 1 219 #else 220 #define _GLIBCXX_SIMD_HAVE_SSE_ABI 0 221 #endif 222 #if _GLIBCXX_SIMD_HAVE_SSE2 223 #define _GLIBCXX_SIMD_HAVE_FULL_SSE_ABI 1 224 #else 225 #define _GLIBCXX_SIMD_HAVE_FULL_SSE_ABI 0 226 #endif 227 228 #if _GLIBCXX_SIMD_HAVE_AVX 229 #define _GLIBCXX_SIMD_HAVE_AVX_ABI 1 230 #else 231 #define _GLIBCXX_SIMD_HAVE_AVX_ABI 0 232 #endif 233 #if _GLIBCXX_SIMD_HAVE_AVX2 234 #define _GLIBCXX_SIMD_HAVE_FULL_AVX_ABI 1 235 #else 236 #define _GLIBCXX_SIMD_HAVE_FULL_AVX_ABI 0 237 #endif 238 239 #if _GLIBCXX_SIMD_HAVE_AVX512F 240 #define _GLIBCXX_SIMD_HAVE_AVX512_ABI 1 241 #else 242 #define _GLIBCXX_SIMD_HAVE_AVX512_ABI 0 243 #endif 244 #if _GLIBCXX_SIMD_HAVE_AVX512BW 245 #define _GLIBCXX_SIMD_HAVE_FULL_AVX512_ABI 1 246 #else 247 #define _GLIBCXX_SIMD_HAVE_FULL_AVX512_ABI 0 248 #endif 249 250 #if defined __x86_64__ && !_GLIBCXX_SIMD_HAVE_SSE2 251 #error "Use of SSE2 is required on AMD64" 252 #endif 253 //}}} 254 255 #ifdef __clang__ 256 #define _GLIBCXX_SIMD_NORMAL_MATH 257 #define _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 258 #else 259 #define _GLIBCXX_SIMD_NORMAL_MATH \ 260 [[__gnu__::__optimize__("finite-math-only,no-signed-zeros")]] 261 #define _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA __attribute__((__always_inline__)) 262 #endif 263 #define _GLIBCXX_SIMD_NEVER_INLINE [[__gnu__::__noinline__]] 264 #define _GLIBCXX_SIMD_INTRINSIC \ 265 [[__gnu__::__always_inline__, __gnu__::__artificial__]] inline 266 #define _GLIBCXX_SIMD_ALWAYS_INLINE [[__gnu__::__always_inline__]] inline 267 #define _GLIBCXX_SIMD_IS_UNLIKELY(__x) __builtin_expect(__x, 0) 268 #define _GLIBCXX_SIMD_IS_LIKELY(__x) __builtin_expect(__x, 1) 269 270 #if __STRICT_ANSI__ || defined __clang__ 271 #define _GLIBCXX_SIMD_CONSTEXPR 272 #define _GLIBCXX_SIMD_USE_CONSTEXPR_API const 273 #else 274 #define _GLIBCXX_SIMD_CONSTEXPR constexpr 275 #define _GLIBCXX_SIMD_USE_CONSTEXPR_API constexpr 276 #endif 277 278 #if defined __clang__ 279 #define _GLIBCXX_SIMD_USE_CONSTEXPR const 280 #else 281 #define _GLIBCXX_SIMD_USE_CONSTEXPR constexpr 282 #endif 283 284 #define _GLIBCXX_SIMD_LIST_BINARY(__macro) __macro(|) __macro(&) __macro(^) 285 #define _GLIBCXX_SIMD_LIST_SHIFTS(__macro) __macro(<<) __macro(>>) 286 #define _GLIBCXX_SIMD_LIST_ARITHMETICS(__macro) \ 287 __macro(+) __macro(-) __macro(*) __macro(/) __macro(%) 288 289 #define _GLIBCXX_SIMD_ALL_BINARY(__macro) \ 290 _GLIBCXX_SIMD_LIST_BINARY(__macro) static_assert(true) 291 #define _GLIBCXX_SIMD_ALL_SHIFTS(__macro) \ 292 _GLIBCXX_SIMD_LIST_SHIFTS(__macro) static_assert(true) 293 #define _GLIBCXX_SIMD_ALL_ARITHMETICS(__macro) \ 294 _GLIBCXX_SIMD_LIST_ARITHMETICS(__macro) static_assert(true) 295 296 #ifdef _GLIBCXX_SIMD_NO_ALWAYS_INLINE 297 #undef _GLIBCXX_SIMD_ALWAYS_INLINE 298 #define _GLIBCXX_SIMD_ALWAYS_INLINE inline 299 #undef _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 300 #define _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 301 #undef _GLIBCXX_SIMD_INTRINSIC 302 #define _GLIBCXX_SIMD_INTRINSIC inline 303 #endif 304 305 #if _GLIBCXX_SIMD_HAVE_SSE || _GLIBCXX_SIMD_HAVE_MMX 306 #define _GLIBCXX_SIMD_X86INTRIN 1 307 #else 308 #define _GLIBCXX_SIMD_X86INTRIN 0 309 #endif 310 311 // workaround macros {{{ 312 // use aliasing loads to help GCC understand the data accesses better 313 // This also seems to hide a miscompilation on swap(x[i], x[i + 1]) with 314 // fixed_size_simd<float, 16> x. 315 #define _GLIBCXX_SIMD_USE_ALIASING_LOADS 1 316 317 // vector conversions on x86 not optimized: 318 #if _GLIBCXX_SIMD_X86INTRIN 319 #define _GLIBCXX_SIMD_WORKAROUND_PR85048 1 320 #endif 321 322 // integer division not optimized 323 #ifndef __clang__ 324 #define _GLIBCXX_SIMD_WORKAROUND_PR90993 1 325 #endif 326 327 // very bad codegen for extraction and concatenation of 128/256 "subregisters" 328 // with sizeof(element type) < 8: https://godbolt.org/g/mqUsgM 329 #if _GLIBCXX_SIMD_X86INTRIN 330 #define _GLIBCXX_SIMD_WORKAROUND_XXX_1 1 331 #endif 332 333 // bad codegen for 8 Byte memcpy to __vector_type_t<char, 16> 334 #define _GLIBCXX_SIMD_WORKAROUND_PR90424 1 335 336 // bad codegen for zero-extend using simple concat(__x, 0) 337 #if _GLIBCXX_SIMD_X86INTRIN 338 #define _GLIBCXX_SIMD_WORKAROUND_XXX_3 1 339 #endif 340 341 // https://github.com/cplusplus/parallelism-ts/issues/65 (incorrect return type 342 // of static_simd_cast) 343 #define _GLIBCXX_SIMD_FIX_P2TS_ISSUE65 1 344 345 // https://github.com/cplusplus/parallelism-ts/issues/66 (incorrect SFINAE 346 // constraint on (static)_simd_cast) 347 #define _GLIBCXX_SIMD_FIX_P2TS_ISSUE66 1 348 // }}} 349 350 /// @endcond 351 352 #endif // __cplusplus >= 201703L 353 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_DETAIL_H_ 354 355 // vim: foldmethod=marker 356