1 // -*- C++ -*- 2 //===----------------------------------------------------------------------===// 3 // 4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5 // See https://llvm.org/LICENSE.txt for license information. 6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7 // 8 //===----------------------------------------------------------------------===// 9 10 #ifndef _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H 11 #define _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H 12 13 #include <__assert> 14 #include <__bit/bit_ceil.h> 15 #include <__config> 16 #include <__cstddef/size_t.h> 17 #include <__type_traits/integral_constant.h> 18 #include <__utility/forward.h> 19 #include <__utility/integer_sequence.h> 20 #include <experimental/__simd/declaration.h> 21 #include <experimental/__simd/traits.h> 22 #include <experimental/__simd/utility.h> 23 24 #if _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) 25 26 _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL 27 inline namespace parallelism_v2 { 28 namespace simd_abi { 29 template <int _Np> 30 struct __vec_ext { 31 static constexpr size_t __simd_size = _Np; 32 }; 33 } // namespace simd_abi 34 35 template <int _Np> 36 inline constexpr bool is_abi_tag_v<simd_abi::__vec_ext<_Np>> = _Np > 0 && _Np <= 32; 37 38 template <class _Tp, int _Np> 39 struct __simd_storage<_Tp, simd_abi::__vec_ext<_Np>> { 40 _Tp __data __attribute__((__vector_size__(std::__bit_ceil((sizeof(_Tp) * _Np))))); 41 42 _LIBCPP_HIDE_FROM_ABI _Tp __get(size_t __idx) const noexcept { 43 _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx < _Np, "Index is out of bounds"); 44 return __data[__idx]; 45 } 46 _LIBCPP_HIDE_FROM_ABI void __set(size_t __idx, _Tp __v) noexcept { 47 _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__idx < _Np, "Index is out of bounds"); 48 __data[__idx] = __v; 49 } 50 }; 51 52 template <class _Tp, int _Np> 53 struct __mask_storage<_Tp, simd_abi::__vec_ext<_Np>> 54 : __simd_storage<decltype(experimental::__choose_mask_type<_Tp>()), simd_abi::__vec_ext<_Np>> {}; 55 56 template <class _Tp, int _Np> 57 struct __simd_operations<_Tp, simd_abi::__vec_ext<_Np>> { 58 using _SimdStorage _LIBCPP_NODEBUG = __simd_storage<_Tp, simd_abi::__vec_ext<_Np>>; 59 using _MaskStorage _LIBCPP_NODEBUG = __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>; 60 61 static _LIBCPP_HIDE_FROM_ABI _SimdStorage __broadcast(_Tp __v) noexcept { 62 _SimdStorage __result; 63 for (int __i = 0; __i < _Np; ++__i) { 64 __result.__set(__i, __v); 65 } 66 return __result; 67 } 68 69 template <class _Generator, size_t... _Is> 70 static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate_init(_Generator&& __g, std::index_sequence<_Is...>) { 71 return _SimdStorage{{__g(std::integral_constant<size_t, _Is>())...}}; 72 } 73 74 template <class _Generator> 75 static _LIBCPP_HIDE_FROM_ABI _SimdStorage __generate(_Generator&& __g) noexcept { 76 return __generate_init(std::forward<_Generator>(__g), std::make_index_sequence<_Np>()); 77 } 78 79 template <class _Up> 80 static _LIBCPP_HIDE_FROM_ABI void __load(_SimdStorage& __s, const _Up* __mem) noexcept { 81 for (size_t __i = 0; __i < _Np; __i++) 82 __s.__data[__i] = static_cast<_Tp>(__mem[__i]); 83 } 84 85 template <class _Up> 86 static _LIBCPP_HIDE_FROM_ABI void __store(_SimdStorage __s, _Up* __mem) noexcept { 87 for (size_t __i = 0; __i < _Np; __i++) 88 __mem[__i] = static_cast<_Up>(__s.__data[__i]); 89 } 90 91 static _LIBCPP_HIDE_FROM_ABI void __increment(_SimdStorage& __s) noexcept { __s.__data = __s.__data + 1; } 92 93 static _LIBCPP_HIDE_FROM_ABI void __decrement(_SimdStorage& __s) noexcept { __s.__data = __s.__data - 1; } 94 95 static _LIBCPP_HIDE_FROM_ABI _MaskStorage __negate(_SimdStorage __s) noexcept { return {!__s.__data}; } 96 97 static _LIBCPP_HIDE_FROM_ABI _SimdStorage __bitwise_not(_SimdStorage __s) noexcept { return {~__s.__data}; } 98 99 static _LIBCPP_HIDE_FROM_ABI _SimdStorage __unary_minus(_SimdStorage __s) noexcept { return {-__s.__data}; } 100 }; 101 102 template <class _Tp, int _Np> 103 struct __mask_operations<_Tp, simd_abi::__vec_ext<_Np>> { 104 using _MaskStorage _LIBCPP_NODEBUG = __mask_storage<_Tp, simd_abi::__vec_ext<_Np>>; 105 106 static _LIBCPP_HIDE_FROM_ABI _MaskStorage __broadcast(bool __v) noexcept { 107 _MaskStorage __result; 108 auto __all_bits_v = experimental::__set_all_bits<_Tp>(__v); 109 for (int __i = 0; __i < _Np; ++__i) { 110 __result.__set(__i, __all_bits_v); 111 } 112 return __result; 113 } 114 115 static _LIBCPP_HIDE_FROM_ABI void __load(_MaskStorage& __s, const bool* __mem) noexcept { 116 for (size_t __i = 0; __i < _Np; __i++) 117 __s.__data[__i] = experimental::__set_all_bits<_Tp>(__mem[__i]); 118 } 119 120 static _LIBCPP_HIDE_FROM_ABI void __store(_MaskStorage __s, bool* __mem) noexcept { 121 for (size_t __i = 0; __i < _Np; __i++) 122 __mem[__i] = static_cast<bool>(__s.__data[__i]); 123 } 124 }; 125 126 } // namespace parallelism_v2 127 _LIBCPP_END_NAMESPACE_EXPERIMENTAL 128 129 #endif // _LIBCPP_STD_VER >= 17 && defined(_LIBCPP_ENABLE_EXPERIMENTAL) 130 #endif // _LIBCPP_EXPERIMENTAL___SIMD_VEC_EXT_H 131