1 //===-- Unittests for op_ files -------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "memory_check_utils.h" 10 #include "src/__support/macros/config.h" 11 #include "src/__support/macros/properties/os.h" 12 #include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT64 13 #include "src/string/memory_utils/op_aarch64.h" 14 #include "src/string/memory_utils/op_builtin.h" 15 #include "src/string/memory_utils/op_generic.h" 16 #include "src/string/memory_utils/op_riscv.h" 17 #include "src/string/memory_utils/op_x86.h" 18 #include "test/UnitTest/Test.h" 19 20 namespace LIBC_NAMESPACE_DECL { 21 22 template <typename T> struct has_head_tail { 23 template <typename C> static char sfinae(decltype(&C::head_tail)); 24 template <typename C> static uint16_t sfinae(...); 25 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char); 26 }; 27 28 template <typename T> struct has_loop_and_tail { 29 template <typename C> static char sfinae(decltype(&C::loop_and_tail)); 30 template <typename C> static uint16_t sfinae(...); 31 static constexpr bool value = sizeof(sfinae<T>(0)) == sizeof(char); 32 }; 33 34 // Allocates two Buffer and extracts two spans out of them, one 35 // aligned and one misaligned. Tests are run on both spans. 36 struct Buffers { 37 Buffers(size_t size) 38 : aligned_buffer(size, Aligned::YES), 39 misaligned_buffer(size, Aligned::NO) {} 40 41 // Returns two spans of 'size' bytes. The first is aligned on 42 // Buffer::kAlign and the second one is unaligned. 43 cpp::array<cpp::span<char>, 2> spans() { 44 return {aligned_buffer.span(), misaligned_buffer.span()}; 45 } 46 47 Buffer aligned_buffer; 48 Buffer misaligned_buffer; 49 }; 50 51 using MemcpyImplementations = testing::TypeList< 52 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE 53 builtin::Memcpy<1>, // 54 builtin::Memcpy<2>, // 55 builtin::Memcpy<3>, // 56 builtin::Memcpy<4>, // 57 builtin::Memcpy<8>, // 58 builtin::Memcpy<16>, // 59 builtin::Memcpy<32>, // 60 builtin::Memcpy<64> 61 #endif // LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE 62 >; 63 64 // Convenient helper to turn a span into cpp::byte *. 65 static inline cpp::byte *as_byte(cpp::span<char> span) { 66 return reinterpret_cast<cpp::byte *>(span.data()); 67 } 68 69 // Adapt CheckMemcpy signature to op implementation signatures. 70 template <auto FnImpl> 71 void CopyAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) { 72 FnImpl(as_byte(dst), as_byte(src), size); 73 } 74 template <size_t Size, auto FnImpl> 75 void CopyBlockAdaptor(cpp::span<char> dst, cpp::span<char> src, size_t size) { 76 FnImpl(as_byte(dst), as_byte(src)); 77 } 78 79 TYPED_TEST(LlvmLibcOpTest, Memcpy, MemcpyImplementations) { 80 using Impl = ParamType; 81 constexpr size_t kSize = Impl::SIZE; 82 { // Test block operation 83 static constexpr auto BlockImpl = CopyBlockAdaptor<kSize, Impl::block>; 84 Buffers SrcBuffer(kSize); 85 Buffers DstBuffer(kSize); 86 for (auto src : SrcBuffer.spans()) { 87 Randomize(src); 88 for (auto dst : DstBuffer.spans()) { 89 ASSERT_TRUE(CheckMemcpy<BlockImpl>(dst, src, kSize)); 90 } 91 } 92 } 93 { // Test head tail operations from kSize to 2 * kSize. 94 static constexpr auto HeadTailImpl = CopyAdaptor<Impl::head_tail>; 95 Buffer SrcBuffer(2 * kSize); 96 Buffer DstBuffer(2 * kSize); 97 Randomize(SrcBuffer.span()); 98 for (size_t size = kSize; size < 2 * kSize; ++size) { 99 auto src = SrcBuffer.span().subspan(0, size); 100 auto dst = DstBuffer.span().subspan(0, size); 101 ASSERT_TRUE(CheckMemcpy<HeadTailImpl>(dst, src, size)); 102 } 103 } 104 { // Test loop operations from kSize to 3 * kSize. 105 if constexpr (kSize > 1) { 106 static constexpr auto LoopImpl = CopyAdaptor<Impl::loop_and_tail>; 107 Buffer SrcBuffer(3 * kSize); 108 Buffer DstBuffer(3 * kSize); 109 Randomize(SrcBuffer.span()); 110 for (size_t size = kSize; size < 3 * kSize; ++size) { 111 auto src = SrcBuffer.span().subspan(0, size); 112 auto dst = DstBuffer.span().subspan(0, size); 113 ASSERT_TRUE(CheckMemcpy<LoopImpl>(dst, src, size)); 114 } 115 } 116 } 117 } 118 119 using MemsetImplementations = testing::TypeList< 120 #ifdef LLVM_LIBC_HAS_BUILTIN_MEMSET_INLINE 121 builtin::Memset<1>, // 122 builtin::Memset<2>, // 123 builtin::Memset<3>, // 124 builtin::Memset<4>, // 125 builtin::Memset<8>, // 126 builtin::Memset<16>, // 127 builtin::Memset<32>, // 128 builtin::Memset<64>, 129 #endif 130 #ifdef LIBC_TYPES_HAS_INT64 131 generic::Memset<uint64_t>, generic::Memset<cpp::array<uint64_t, 2>>, 132 #endif // LIBC_TYPES_HAS_INT64 133 #ifdef __AVX512F__ 134 generic::Memset<generic_v512>, generic::Memset<cpp::array<generic_v512, 2>>, 135 #endif 136 #ifdef __AVX__ 137 generic::Memset<generic_v256>, generic::Memset<cpp::array<generic_v256, 2>>, 138 #endif 139 #ifdef __SSE2__ 140 generic::Memset<generic_v128>, generic::Memset<cpp::array<generic_v128, 2>>, 141 #endif 142 generic::Memset<uint32_t>, generic::Memset<cpp::array<uint32_t, 2>>, // 143 generic::Memset<uint16_t>, generic::Memset<cpp::array<uint16_t, 2>>, // 144 generic::Memset<uint8_t>, generic::Memset<cpp::array<uint8_t, 2>>, // 145 generic::MemsetSequence<uint8_t, uint8_t>, // 146 generic::MemsetSequence<uint16_t, uint8_t>, // 147 generic::MemsetSequence<uint32_t, uint16_t, uint8_t> // 148 >; 149 150 // Adapt CheckMemset signature to op implementation signatures. 151 template <auto FnImpl> 152 void SetAdaptor(cpp::span<char> dst, uint8_t value, size_t size) { 153 FnImpl(as_byte(dst), value, size); 154 } 155 template <size_t Size, auto FnImpl> 156 void SetBlockAdaptor(cpp::span<char> dst, uint8_t value, size_t size) { 157 FnImpl(as_byte(dst), value); 158 } 159 160 TYPED_TEST(LlvmLibcOpTest, Memset, MemsetImplementations) { 161 using Impl = ParamType; 162 constexpr size_t kSize = Impl::SIZE; 163 { // Test block operation 164 static constexpr auto BlockImpl = SetBlockAdaptor<kSize, Impl::block>; 165 Buffers DstBuffer(kSize); 166 for (uint8_t value : cpp::array<uint8_t, 3>{0, 1, 255}) { 167 for (auto dst : DstBuffer.spans()) { 168 ASSERT_TRUE(CheckMemset<BlockImpl>(dst, value, kSize)); 169 } 170 } 171 } 172 if constexpr (has_head_tail<Impl>::value) { 173 // Test head tail operations from kSize to 2 * kSize. 174 static constexpr auto HeadTailImpl = SetAdaptor<Impl::head_tail>; 175 Buffer DstBuffer(2 * kSize); 176 for (size_t size = kSize; size < 2 * kSize; ++size) { 177 const char value = size % 10; 178 auto dst = DstBuffer.span().subspan(0, size); 179 ASSERT_TRUE(CheckMemset<HeadTailImpl>(dst, value, size)); 180 } 181 } 182 if constexpr (has_loop_and_tail<Impl>::value) { 183 // Test loop operations from kSize to 3 * kSize. 184 if constexpr (kSize > 1) { 185 static constexpr auto LoopImpl = SetAdaptor<Impl::loop_and_tail>; 186 Buffer DstBuffer(3 * kSize); 187 for (size_t size = kSize; size < 3 * kSize; ++size) { 188 const char value = size % 10; 189 auto dst = DstBuffer.span().subspan(0, size); 190 ASSERT_TRUE((CheckMemset<LoopImpl>(dst, value, size))); 191 } 192 } 193 } 194 } 195 196 #ifdef LIBC_TARGET_ARCH_IS_X86_64 197 // Prevent GCC warning due to ignored __aligned__ attributes when passing x86 198 // SIMD types as template arguments. 199 #pragma GCC diagnostic push 200 #pragma GCC diagnostic ignored "-Wignored-attributes" 201 #endif // LIBC_TARGET_ARCH_IS_X86_64 202 203 using BcmpImplementations = testing::TypeList< 204 #ifdef LIBC_TARGET_ARCH_IS_X86_64 205 #ifdef __SSE4_1__ 206 generic::Bcmp<__m128i>, 207 #endif // __SSE4_1__ 208 #ifdef __AVX2__ 209 generic::Bcmp<__m256i>, 210 #endif // __AVX2__ 211 #ifdef __AVX512BW__ 212 generic::Bcmp<__m512i>, 213 #endif // __AVX512BW__ 214 215 #endif // LIBC_TARGET_ARCH_IS_X86_64 216 #ifdef LIBC_TARGET_ARCH_IS_AARCH64 217 aarch64::Bcmp<16>, // 218 aarch64::Bcmp<32>, 219 #endif 220 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM 221 generic::Bcmp<uint16_t>, 222 generic::Bcmp<uint32_t>, // 223 #ifdef LIBC_TYPES_HAS_INT64 224 generic::Bcmp<uint64_t>, 225 #endif // LIBC_TYPES_HAS_INT64 226 generic::BcmpSequence<uint16_t, uint8_t>, 227 generic::BcmpSequence<uint32_t, uint8_t>, // 228 generic::BcmpSequence<uint32_t, uint16_t>, // 229 generic::BcmpSequence<uint32_t, uint16_t, uint8_t>, 230 #endif // LIBC_TARGET_ARCH_IS_ARM 231 generic::BcmpSequence<uint8_t, uint8_t>, 232 generic::BcmpSequence<uint8_t, uint8_t, uint8_t>, // 233 generic::Bcmp<uint8_t>>; 234 235 #ifdef LIBC_TARGET_ARCH_IS_X86_64 236 #pragma GCC diagnostic pop 237 #endif // LIBC_TARGET_ARCH_IS_X86_64 238 239 // Adapt CheckBcmp signature to op implementation signatures. 240 template <auto FnImpl> 241 int CmpAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) { 242 return (int)FnImpl(as_byte(p1), as_byte(p2), size); 243 } 244 template <size_t Size, auto FnImpl> 245 int CmpBlockAdaptor(cpp::span<char> p1, cpp::span<char> p2, size_t size) { 246 return (int)FnImpl(as_byte(p1), as_byte(p2)); 247 } 248 249 TYPED_TEST(LlvmLibcOpTest, Bcmp, BcmpImplementations) { 250 using Impl = ParamType; 251 constexpr size_t kSize = Impl::SIZE; 252 { // Test block operation 253 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>; 254 Buffers Buffer1(kSize); 255 Buffers Buffer2(kSize); 256 for (auto span1 : Buffer1.spans()) { 257 Randomize(span1); 258 for (auto span2 : Buffer2.spans()) 259 ASSERT_TRUE((CheckBcmp<BlockImpl>(span1, span2, kSize))); 260 } 261 } 262 if constexpr (has_head_tail<Impl>::value) { 263 // Test head tail operations from kSize to 2 * kSize. 264 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>; 265 Buffer Buffer1(2 * kSize); 266 Buffer Buffer2(2 * kSize); 267 Randomize(Buffer1.span()); 268 for (size_t size = kSize; size < 2 * kSize; ++size) { 269 auto span1 = Buffer1.span().subspan(0, size); 270 auto span2 = Buffer2.span().subspan(0, size); 271 ASSERT_TRUE((CheckBcmp<HeadTailImpl>(span1, span2, size))); 272 } 273 } 274 if constexpr (has_loop_and_tail<Impl>::value) { 275 // Test loop operations from kSize to 3 * kSize. 276 if constexpr (kSize > 1) { 277 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>; 278 Buffer Buffer1(3 * kSize); 279 Buffer Buffer2(3 * kSize); 280 Randomize(Buffer1.span()); 281 for (size_t size = kSize; size < 3 * kSize; ++size) { 282 auto span1 = Buffer1.span().subspan(0, size); 283 auto span2 = Buffer2.span().subspan(0, size); 284 ASSERT_TRUE((CheckBcmp<LoopImpl>(span1, span2, size))); 285 } 286 } 287 } 288 } 289 290 #ifdef LIBC_TARGET_ARCH_IS_X86_64 291 // Prevent GCC warning due to ignored __aligned__ attributes when passing x86 292 // SIMD types as template arguments. 293 #pragma GCC diagnostic push 294 #pragma GCC diagnostic ignored "-Wignored-attributes" 295 #endif // LIBC_TARGET_ARCH_IS_X86_64 296 297 using MemcmpImplementations = testing::TypeList< 298 #if defined(LIBC_TARGET_ARCH_IS_X86_64) && !defined(LIBC_TARGET_OS_IS_WINDOWS) 299 #ifdef __SSE2__ 300 generic::Memcmp<__m128i>, // 301 #endif 302 #ifdef __AVX2__ 303 generic::Memcmp<__m256i>, // 304 #endif 305 #ifdef __AVX512BW__ 306 generic::Memcmp<__m512i>, // 307 #endif 308 #endif // LIBC_TARGET_ARCH_IS_X86_64 309 #ifdef LIBC_TARGET_ARCH_IS_AARCH64 310 generic::Memcmp<uint8x16_t>, // 311 generic::Memcmp<uint8x16x2_t>, 312 #endif 313 #ifndef LIBC_TARGET_ARCH_IS_ARM // Removing non uint8_t types for ARM 314 generic::Memcmp<uint16_t>, 315 generic::Memcmp<uint32_t>, // 316 #ifdef LIBC_TYPES_HAS_INT64 317 generic::Memcmp<uint64_t>, 318 #endif // LIBC_TYPES_HAS_INT64 319 generic::MemcmpSequence<uint16_t, uint8_t>, 320 generic::MemcmpSequence<uint32_t, uint16_t, uint8_t>, // 321 #endif // LIBC_TARGET_ARCH_IS_ARM 322 generic::MemcmpSequence<uint8_t, uint8_t>, 323 generic::MemcmpSequence<uint8_t, uint8_t, uint8_t>, 324 generic::Memcmp<uint8_t>>; 325 326 #ifdef LIBC_TARGET_ARCH_IS_X86_64 327 #pragma GCC diagnostic pop 328 #endif // LIBC_TARGET_ARCH_IS_X86_64 329 330 TYPED_TEST(LlvmLibcOpTest, Memcmp, MemcmpImplementations) { 331 using Impl = ParamType; 332 constexpr size_t kSize = Impl::SIZE; 333 { // Test block operation 334 static constexpr auto BlockImpl = CmpBlockAdaptor<kSize, Impl::block>; 335 Buffers Buffer1(kSize); 336 Buffers Buffer2(kSize); 337 for (auto span1 : Buffer1.spans()) { 338 Randomize(span1); 339 for (auto span2 : Buffer2.spans()) 340 ASSERT_TRUE((CheckMemcmp<BlockImpl>(span1, span2, kSize))); 341 } 342 } 343 if constexpr (has_head_tail<Impl>::value) { 344 // Test head tail operations from kSize to 2 * kSize. 345 static constexpr auto HeadTailImpl = CmpAdaptor<Impl::head_tail>; 346 Buffer Buffer1(2 * kSize); 347 Buffer Buffer2(2 * kSize); 348 Randomize(Buffer1.span()); 349 for (size_t size = kSize; size < 2 * kSize; ++size) { 350 auto span1 = Buffer1.span().subspan(0, size); 351 auto span2 = Buffer2.span().subspan(0, size); 352 ASSERT_TRUE((CheckMemcmp<HeadTailImpl>(span1, span2, size))); 353 } 354 } 355 if constexpr (has_loop_and_tail<Impl>::value) { 356 // Test loop operations from kSize to 3 * kSize. 357 if constexpr (kSize > 1) { 358 static constexpr auto LoopImpl = CmpAdaptor<Impl::loop_and_tail>; 359 Buffer Buffer1(3 * kSize); 360 Buffer Buffer2(3 * kSize); 361 Randomize(Buffer1.span()); 362 for (size_t size = kSize; size < 3 * kSize; ++size) { 363 auto span1 = Buffer1.span().subspan(0, size); 364 auto span2 = Buffer2.span().subspan(0, size); 365 ASSERT_TRUE((CheckMemcmp<LoopImpl>(span1, span2, size))); 366 } 367 } 368 } 369 } 370 371 } // namespace LIBC_NAMESPACE_DECL 372