1 //===----------------------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef _LIBCPP___ALGORITHM_SIMD_UTILS_H 10 #define _LIBCPP___ALGORITHM_SIMD_UTILS_H 11 12 #include <__bit/bit_cast.h> 13 #include <__bit/countr.h> 14 #include <__config> 15 #include <__type_traits/is_arithmetic.h> 16 #include <__type_traits/is_same.h> 17 #include <__utility/integer_sequence.h> 18 #include <cstddef> 19 #include <cstdint> 20 21 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 22 # pragma GCC system_header 23 #endif 24 25 // TODO: Find out how altivec changes things and allow vectorizations there too. 26 #if _LIBCPP_STD_VER >= 14 && defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1700 && !defined(__ALTIVEC__) 27 # define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 1 28 #else 29 # define _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 0 30 #endif 31 32 #if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS && !defined(__OPTIMIZE_SIZE__) 33 # define _LIBCPP_VECTORIZE_ALGORITHMS 1 34 #else 35 # define _LIBCPP_VECTORIZE_ALGORITHMS 0 36 #endif 37 38 #if _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 39 40 _LIBCPP_BEGIN_NAMESPACE_STD 41 42 // This isn't specialized for 64 byte vectors on purpose. They have the potential to significantly reduce performance 43 // in mixed simd/non-simd workloads and don't provide any performance improvement for currently vectorized algorithms 44 // as far as benchmarks are concerned. 45 # if defined(__AVX__) 46 template <class _Tp> 47 inline constexpr size_t __native_vector_size = 32 / sizeof(_Tp); 48 # elif defined(__SSE__) || defined(__ARM_NEON__) 49 template <class _Tp> 50 inline constexpr size_t __native_vector_size = 16 / sizeof(_Tp); 51 # elif defined(__MMX__) 52 template <class _Tp> 53 inline constexpr size_t __native_vector_size = 8 / sizeof(_Tp); 54 # else 55 template <class _Tp> 56 inline constexpr size_t __native_vector_size = 1; 57 # endif 58 59 template <class _ArithmeticT, size_t _Np> 60 using __simd_vector __attribute__((__ext_vector_type__(_Np))) = _ArithmeticT; 61 62 template <class _VecT> 63 inline constexpr size_t __simd_vector_size_v = []<bool _False = false>() -> size_t { 64 static_assert(_False, "Not a vector!"); 65 }(); 66 67 template <class _Tp, size_t _Np> 68 inline constexpr size_t __simd_vector_size_v<__simd_vector<_Tp, _Np>> = _Np; 69 70 template <class _Tp, size_t _Np> 71 _LIBCPP_HIDE_FROM_ABI _Tp __simd_vector_underlying_type_impl(__simd_vector<_Tp, _Np>) { 72 return _Tp{}; 73 } 74 75 template <class _VecT> 76 using __simd_vector_underlying_type_t = decltype(std::__simd_vector_underlying_type_impl(_VecT{})); 77 78 // This isn't inlined without always_inline when loading chars. 79 template <class _VecT, class _Tp> 80 _LIBCPP_NODISCARD _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _VecT __load_vector(const _Tp* __ptr) noexcept { 81 return [=]<size_t... _Indices>(index_sequence<_Indices...>) _LIBCPP_ALWAYS_INLINE noexcept { 82 return _VecT{__ptr[_Indices]...}; 83 }(make_index_sequence<__simd_vector_size_v<_VecT>>{}); 84 } 85 86 template <class _Tp, size_t _Np> 87 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI bool __all_of(__simd_vector<_Tp, _Np> __vec) noexcept { 88 return __builtin_reduce_and(__builtin_convertvector(__vec, __simd_vector<bool, _Np>)); 89 } 90 91 template <class _Tp, size_t _Np> 92 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_set(__simd_vector<_Tp, _Np> __vec) noexcept { 93 using __mask_vec = __simd_vector<bool, _Np>; 94 95 // This has MSan disabled du to https://github.com/llvm/llvm-project/issues/85876 96 auto __impl = [&]<class _MaskT>(_MaskT) _LIBCPP_NO_SANITIZE("memory") noexcept { 97 return std::__countr_zero(__builtin_bit_cast(_MaskT, __builtin_convertvector(__vec, __mask_vec))); 98 }; 99 100 if constexpr (sizeof(__mask_vec) == sizeof(uint8_t)) { 101 return __impl(uint8_t{}); 102 } else if constexpr (sizeof(__mask_vec) == sizeof(uint16_t)) { 103 return __impl(uint16_t{}); 104 } else if constexpr (sizeof(__mask_vec) == sizeof(uint32_t)) { 105 return __impl(uint32_t{}); 106 } else if constexpr (sizeof(__mask_vec) == sizeof(uint64_t)) { 107 return __impl(uint64_t{}); 108 } else { 109 static_assert(sizeof(__mask_vec) == 0, "unexpected required size for mask integer type"); 110 return 0; 111 } 112 } 113 114 template <class _Tp, size_t _Np> 115 _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI size_t __find_first_not_set(__simd_vector<_Tp, _Np> __vec) noexcept { 116 return std::__find_first_set(~__vec); 117 } 118 119 _LIBCPP_END_NAMESPACE_STD 120 121 #endif // _LIBCPP_HAS_ALGORITHM_VECTOR_UTILS 122 123 #endif // _LIBCPP___ALGORITHM_SIMD_UTILS_H 124