1 // Generic simd conversions -*- C++ -*- 2 3 // Copyright (C) 2020-2022 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ 26 #define _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ 27 28 #if __cplusplus >= 201703L 29 30 _GLIBCXX_SIMD_BEGIN_NAMESPACE 31 // _SimdConverter scalar -> scalar {{{ 32 template <typename _From, typename _To> 33 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::scalar, 34 enable_if_t<!is_same_v<_From, _To>>> 35 { 36 _GLIBCXX_SIMD_INTRINSIC constexpr _To operator()(_From __a) const noexcept 37 { return static_cast<_To>(__a); } 38 }; 39 40 // }}} 41 // _SimdConverter scalar -> "native" {{{ 42 template <typename _From, typename _To, typename _Abi> 43 struct _SimdConverter<_From, simd_abi::scalar, _To, _Abi, 44 enable_if_t<!is_same_v<_Abi, simd_abi::scalar>>> 45 { 46 using _Ret = typename _Abi::template __traits<_To>::_SimdMember; 47 48 template <typename... _More> 49 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 50 operator()(_From __a, _More... __more) const noexcept 51 { 52 static_assert(sizeof...(_More) + 1 == _Abi::template _S_size<_To>); 53 static_assert(conjunction_v<is_same<_From, _More>...>); 54 return __make_vector<_To>(__a, __more...); 55 } 56 }; 57 58 // }}} 59 // _SimdConverter "native 1" -> "native 2" {{{ 60 template <typename _From, typename _To, typename _AFrom, typename _ATo> 61 struct _SimdConverter< 62 _From, _AFrom, _To, _ATo, 63 enable_if_t<!disjunction_v< 64 __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>, 65 is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>, 66 conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>>>> 67 { 68 using _Arg = typename _AFrom::template __traits<_From>::_SimdMember; 69 using _Ret = typename _ATo::template __traits<_To>::_SimdMember; 70 using _V = __vector_type_t<_To, simd_size_v<_To, _ATo>>; 71 72 template <typename... _More> 73 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 74 operator()(_Arg __a, _More... __more) const noexcept 75 { return __vector_convert<_V>(__a, __more...); } 76 }; 77 78 // }}} 79 // _SimdConverter scalar -> fixed_size<1> {{{1 80 template <typename _From, typename _To> 81 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::fixed_size<1>, 82 void> 83 { 84 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_To, simd_abi::scalar> 85 operator()(_From __x) const noexcept 86 { return {static_cast<_To>(__x)}; } 87 }; 88 89 // _SimdConverter fixed_size<1> -> scalar {{{1 90 template <typename _From, typename _To> 91 struct _SimdConverter<_From, simd_abi::fixed_size<1>, _To, simd_abi::scalar, 92 void> 93 { 94 _GLIBCXX_SIMD_INTRINSIC constexpr _To 95 operator()(_SimdTuple<_From, simd_abi::scalar> __x) const noexcept 96 { return {static_cast<_To>(__x.first)}; } 97 }; 98 99 // _SimdConverter fixed_size<_Np> -> fixed_size<_Np> {{{1 100 template <typename _From, typename _To, int _Np> 101 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, 102 simd_abi::fixed_size<_Np>, 103 enable_if_t<!is_same_v<_From, _To>>> 104 { 105 using _Ret = __fixed_size_storage_t<_To, _Np>; 106 using _Arg = __fixed_size_storage_t<_From, _Np>; 107 108 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 109 operator()(const _Arg& __x) const noexcept 110 { 111 if constexpr (is_same_v<_From, _To>) 112 return __x; 113 114 // special case (optimize) int signedness casts 115 else if constexpr (sizeof(_From) == sizeof(_To) 116 && is_integral_v<_From> && is_integral_v<_To>) 117 return __bit_cast<_Ret>(__x); 118 119 // special case if all ABI tags in _Ret are scalar 120 else if constexpr (__is_scalar_abi<typename _Ret::_FirstAbi>()) 121 { 122 return __call_with_subscripts( 123 __x, make_index_sequence<_Np>(), 124 [](auto... __values) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Ret { 125 return __make_simd_tuple<_To, decltype((void) __values, 126 simd_abi::scalar())...>( 127 static_cast<_To>(__values)...); 128 }); 129 } 130 131 // from one vector to one vector 132 else if constexpr (_Arg::_S_first_size == _Ret::_S_first_size) 133 { 134 _SimdConverter<_From, typename _Arg::_FirstAbi, _To, 135 typename _Ret::_FirstAbi> 136 __native_cvt; 137 if constexpr (_Arg::_S_tuple_size == 1) 138 return {__native_cvt(__x.first)}; 139 else 140 { 141 constexpr size_t _NRemain = _Np - _Arg::_S_first_size; 142 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To, 143 simd_abi::fixed_size<_NRemain>> 144 __remainder_cvt; 145 return {__native_cvt(__x.first), __remainder_cvt(__x.second)}; 146 } 147 } 148 149 // from one vector to multiple vectors 150 else if constexpr (_Arg::_S_first_size > _Ret::_S_first_size) 151 { 152 const auto __multiple_return_chunks 153 = __convert_all<__vector_type_t<_To, _Ret::_S_first_size>>( 154 __x.first); 155 constexpr auto __converted = __multiple_return_chunks.size() 156 * _Ret::_FirstAbi::template _S_size<_To>; 157 constexpr auto __remaining = _Np - __converted; 158 if constexpr (_Arg::_S_tuple_size == 1 && __remaining == 0) 159 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks); 160 else if constexpr (_Arg::_S_tuple_size == 1) 161 { // e.g. <int, 3> -> <double, 2, 1> or <short, 7> -> <double, 4, 2, 162 // 1> 163 using _RetRem 164 = __remove_cvref_t<decltype(__simd_tuple_pop_front<__converted>( 165 _Ret()))>; 166 const auto __return_chunks2 167 = __convert_all<__vector_type_t<_To, _RetRem::_S_first_size>, 0, 168 __converted>(__x.first); 169 constexpr auto __converted2 170 = __converted 171 + __return_chunks2.size() * _RetRem::_S_first_size; 172 if constexpr (__converted2 == _Np) 173 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks, 174 __return_chunks2); 175 else 176 { 177 using _RetRem2 = __remove_cvref_t< 178 decltype(__simd_tuple_pop_front<__return_chunks2.size() 179 * _RetRem::_S_first_size>( 180 _RetRem()))>; 181 const auto __return_chunks3 = __convert_all< 182 __vector_type_t<_To, _RetRem2::_S_first_size>, 0, 183 __converted2>(__x.first); 184 constexpr auto __converted3 185 = __converted2 186 + __return_chunks3.size() * _RetRem2::_S_first_size; 187 if constexpr (__converted3 == _Np) 188 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks, 189 __return_chunks2, 190 __return_chunks3); 191 else 192 { 193 using _RetRem3 194 = __remove_cvref_t<decltype(__simd_tuple_pop_front< 195 __return_chunks3.size() 196 * _RetRem2::_S_first_size>( 197 _RetRem2()))>; 198 const auto __return_chunks4 = __convert_all< 199 __vector_type_t<_To, _RetRem3::_S_first_size>, 0, 200 __converted3>(__x.first); 201 constexpr auto __converted4 202 = __converted3 203 + __return_chunks4.size() * _RetRem3::_S_first_size; 204 if constexpr (__converted4 == _Np) 205 return __to_simd_tuple<_To, _Np>( 206 __multiple_return_chunks, __return_chunks2, 207 __return_chunks3, __return_chunks4); 208 else 209 __assert_unreachable<_To>(); 210 } 211 } 212 } 213 else 214 { 215 constexpr size_t _NRemain = _Np - _Arg::_S_first_size; 216 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To, 217 simd_abi::fixed_size<_NRemain>> 218 __remainder_cvt; 219 return __simd_tuple_concat( 220 __to_simd_tuple<_To, _Arg::_S_first_size>( 221 __multiple_return_chunks), 222 __remainder_cvt(__x.second)); 223 } 224 } 225 226 // from multiple vectors to one vector 227 // _Arg::_S_first_size < _Ret::_S_first_size 228 // a) heterogeneous input at the end of the tuple (possible with partial 229 // native registers in _Ret) 230 else if constexpr (_Ret::_S_tuple_size == 1 231 && _Np % _Arg::_S_first_size != 0) 232 { 233 static_assert(_Ret::_FirstAbi::template _S_is_partial<_To>); 234 return _Ret{__generate_from_n_evaluations< 235 _Np, typename _VectorTraits<typename _Ret::_FirstType>::type>( 236 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 237 return static_cast<_To>(__x[__i]); 238 })}; 239 } 240 else 241 { 242 static_assert(_Arg::_S_tuple_size > 1); 243 constexpr auto __n 244 = __div_roundup(_Ret::_S_first_size, _Arg::_S_first_size); 245 return __call_with_n_evaluations<__n>( 246 [&__x](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 247 // assuming _Arg Abi tags for all __i are _Arg::_FirstAbi 248 _SimdConverter<_From, typename _Arg::_FirstAbi, _To, 249 typename _Ret::_FirstAbi> 250 __native_cvt; 251 if constexpr (_Ret::_S_tuple_size == 1) 252 return _Ret{__native_cvt(__uncvted...)}; 253 else 254 return _Ret{ 255 __native_cvt(__uncvted...), 256 _SimdConverter< 257 _From, simd_abi::fixed_size<_Np - _Ret::_S_first_size>, _To, 258 simd_abi::fixed_size<_Np - _Ret::_S_first_size>>()( 259 __simd_tuple_pop_front<_Ret::_S_first_size>(__x))}; 260 }, [&__x](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 261 return __get_tuple_at<__i>(__x); 262 }); 263 } 264 } 265 }; 266 267 // _SimdConverter "native" -> fixed_size<_Np> {{{1 268 // i.e. 1 register to ? registers 269 template <typename _From, typename _Ap, typename _To, int _Np> 270 struct _SimdConverter<_From, _Ap, _To, simd_abi::fixed_size<_Np>, 271 enable_if_t<!__is_fixed_size_abi_v<_Ap>>> 272 { 273 static_assert( 274 _Np == simd_size_v<_From, _Ap>, 275 "_SimdConverter to fixed_size only works for equal element counts"); 276 277 using _Ret = __fixed_size_storage_t<_To, _Np>; 278 279 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 280 operator()(typename _SimdTraits<_From, _Ap>::_SimdMember __x) const noexcept 281 { 282 if constexpr (_Ret::_S_tuple_size == 1) 283 return {__vector_convert<typename _Ret::_FirstType::_BuiltinType>(__x)}; 284 else 285 { 286 using _FixedNp = simd_abi::fixed_size<_Np>; 287 _SimdConverter<_From, _FixedNp, _To, _FixedNp> __fixed_cvt; 288 using _FromFixedStorage = __fixed_size_storage_t<_From, _Np>; 289 if constexpr (_FromFixedStorage::_S_tuple_size == 1) 290 return __fixed_cvt(_FromFixedStorage{__x}); 291 else if constexpr (_FromFixedStorage::_S_tuple_size == 2) 292 { 293 _FromFixedStorage __tmp; 294 static_assert(sizeof(__tmp) <= sizeof(__x)); 295 __builtin_memcpy(&__tmp.first, &__x, sizeof(__tmp.first)); 296 __builtin_memcpy(&__tmp.second.first, 297 reinterpret_cast<const char*>(&__x) 298 + sizeof(__tmp.first), 299 sizeof(__tmp.second.first)); 300 return __fixed_cvt(__tmp); 301 } 302 else 303 __assert_unreachable<_From>(); 304 } 305 } 306 }; 307 308 // _SimdConverter fixed_size<_Np> -> "native" {{{1 309 // i.e. ? register to 1 registers 310 template <typename _From, int _Np, typename _To, typename _Ap> 311 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, _Ap, 312 enable_if_t<!__is_fixed_size_abi_v<_Ap>>> 313 { 314 static_assert( 315 _Np == simd_size_v<_To, _Ap>, 316 "_SimdConverter to fixed_size only works for equal element counts"); 317 318 using _Arg = __fixed_size_storage_t<_From, _Np>; 319 320 _GLIBCXX_SIMD_INTRINSIC constexpr 321 typename _SimdTraits<_To, _Ap>::_SimdMember 322 operator()(const _Arg& __x) const noexcept 323 { 324 if constexpr (_Arg::_S_tuple_size == 1) 325 return __vector_convert<__vector_type_t<_To, _Np>>(__x.first); 326 else if constexpr (_Arg::_S_is_homogeneous) 327 return __call_with_n_evaluations<_Arg::_S_tuple_size>( 328 [](auto... __members) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 329 if constexpr ((is_convertible_v<decltype(__members), _To> && ...)) 330 return __vector_type_t<_To, _Np>{static_cast<_To>(__members)...}; 331 else 332 return __vector_convert<__vector_type_t<_To, _Np>>(__members...); 333 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 334 return __get_tuple_at<__i>(__x); 335 }); 336 else if constexpr (__fixed_size_storage_t<_To, _Np>::_S_tuple_size == 1) 337 { 338 _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, 339 simd_abi::fixed_size<_Np>> 340 __fixed_cvt; 341 return __fixed_cvt(__x).first; 342 } 343 else 344 { 345 const _SimdWrapper<_From, _Np> __xv 346 = __generate_from_n_evaluations<_Np, __vector_type_t<_From, _Np>>( 347 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }); 348 return __vector_convert<__vector_type_t<_To, _Np>>(__xv); 349 } 350 } 351 }; 352 353 // }}}1 354 _GLIBCXX_SIMD_END_NAMESPACE 355 #endif // __cplusplus >= 201703L 356 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ 357 358 // vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80 359