xref: /freebsd-src/contrib/llvm-project/llvm/lib/Support/BLAKE3/blake3_impl.h (revision 5f757f3ff9144b609b3c433dfd370cc6bdc191ad)
181ad6265SDimitry Andric #ifndef BLAKE3_IMPL_H
281ad6265SDimitry Andric #define BLAKE3_IMPL_H
381ad6265SDimitry Andric 
481ad6265SDimitry Andric #include <assert.h>
581ad6265SDimitry Andric #include <stdbool.h>
681ad6265SDimitry Andric #include <stddef.h>
781ad6265SDimitry Andric #include <stdint.h>
881ad6265SDimitry Andric #include <string.h>
981ad6265SDimitry Andric 
1081ad6265SDimitry Andric #include "llvm-c/blake3.h"
1181ad6265SDimitry Andric // For \p LLVM_LIBRARY_VISIBILITY
1281ad6265SDimitry Andric #include "llvm/Support/Compiler.h"
1381ad6265SDimitry Andric 
141ac55f4cSDimitry Andric #include "llvm_blake3_prefix.h"
1581ad6265SDimitry Andric 
1681ad6265SDimitry Andric // internal flags
1781ad6265SDimitry Andric enum blake3_flags {
1881ad6265SDimitry Andric   CHUNK_START         = 1 << 0,
1981ad6265SDimitry Andric   CHUNK_END           = 1 << 1,
2081ad6265SDimitry Andric   PARENT              = 1 << 2,
2181ad6265SDimitry Andric   ROOT                = 1 << 3,
2281ad6265SDimitry Andric   KEYED_HASH          = 1 << 4,
2381ad6265SDimitry Andric   DERIVE_KEY_CONTEXT  = 1 << 5,
2481ad6265SDimitry Andric   DERIVE_KEY_MATERIAL = 1 << 6,
2581ad6265SDimitry Andric };
2681ad6265SDimitry Andric 
2781ad6265SDimitry Andric // This C implementation tries to support recent versions of GCC, Clang, and
2881ad6265SDimitry Andric // MSVC.
2981ad6265SDimitry Andric #if defined(_MSC_VER)
3081ad6265SDimitry Andric #define INLINE static __forceinline
3181ad6265SDimitry Andric #else
3281ad6265SDimitry Andric #define INLINE static inline __attribute__((always_inline))
3381ad6265SDimitry Andric #endif
3481ad6265SDimitry Andric 
3581ad6265SDimitry Andric #if defined(__x86_64__) || defined(_M_X64)
3681ad6265SDimitry Andric #define IS_X86
3781ad6265SDimitry Andric #define IS_X86_64
3881ad6265SDimitry Andric #endif
3981ad6265SDimitry Andric 
4081ad6265SDimitry Andric #if defined(__i386__) || defined(_M_IX86)
4181ad6265SDimitry Andric #define IS_X86
4281ad6265SDimitry Andric #define IS_X86_32
4381ad6265SDimitry Andric #endif
4481ad6265SDimitry Andric 
4581ad6265SDimitry Andric #if defined(__aarch64__) || defined(_M_ARM64)
4681ad6265SDimitry Andric #define IS_AARCH64
4781ad6265SDimitry Andric #endif
4881ad6265SDimitry Andric 
4981ad6265SDimitry Andric #if defined(IS_X86)
5081ad6265SDimitry Andric #if defined(_MSC_VER)
5181ad6265SDimitry Andric #include <intrin.h>
5281ad6265SDimitry Andric #endif
5381ad6265SDimitry Andric #include <immintrin.h>
5481ad6265SDimitry Andric #endif
5581ad6265SDimitry Andric 
5681ad6265SDimitry Andric #if !defined(BLAKE3_USE_NEON)
57*5f757f3fSDimitry Andric   // If BLAKE3_USE_NEON not manually set, autodetect based on
58*5f757f3fSDimitry Andric   // AArch64ness and endianness.
59*5f757f3fSDimitry Andric   #if defined(IS_AARCH64) && !defined(__ARM_BIG_ENDIAN)
6081ad6265SDimitry Andric     #define BLAKE3_USE_NEON 1
6181ad6265SDimitry Andric   #else
6281ad6265SDimitry Andric     #define BLAKE3_USE_NEON 0
6381ad6265SDimitry Andric   #endif
6481ad6265SDimitry Andric #endif
6581ad6265SDimitry Andric 
6681ad6265SDimitry Andric #if defined(IS_X86)
6781ad6265SDimitry Andric #define MAX_SIMD_DEGREE 16
6881ad6265SDimitry Andric #elif BLAKE3_USE_NEON == 1
6981ad6265SDimitry Andric #define MAX_SIMD_DEGREE 4
7081ad6265SDimitry Andric #else
7181ad6265SDimitry Andric #define MAX_SIMD_DEGREE 1
7281ad6265SDimitry Andric #endif
7381ad6265SDimitry Andric 
7481ad6265SDimitry Andric // There are some places where we want a static size that's equal to the
7581ad6265SDimitry Andric // MAX_SIMD_DEGREE, but also at least 2.
7681ad6265SDimitry Andric #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
7781ad6265SDimitry Andric 
7881ad6265SDimitry Andric static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
7981ad6265SDimitry Andric                                0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
8081ad6265SDimitry Andric                                0x1F83D9ABUL, 0x5BE0CD19UL};
8181ad6265SDimitry Andric 
8281ad6265SDimitry Andric static const uint8_t MSG_SCHEDULE[7][16] = {
8381ad6265SDimitry Andric     {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
8481ad6265SDimitry Andric     {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
8581ad6265SDimitry Andric     {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
8681ad6265SDimitry Andric     {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
8781ad6265SDimitry Andric     {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
8881ad6265SDimitry Andric     {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
8981ad6265SDimitry Andric     {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
9081ad6265SDimitry Andric };
9181ad6265SDimitry Andric 
9281ad6265SDimitry Andric /* Find index of the highest set bit */
9381ad6265SDimitry Andric /* x is assumed to be nonzero.       */
highest_one(uint64_t x)9481ad6265SDimitry Andric static unsigned int highest_one(uint64_t x) {
9581ad6265SDimitry Andric #if defined(__GNUC__) || defined(__clang__)
9681ad6265SDimitry Andric   return 63 ^ __builtin_clzll(x);
9781ad6265SDimitry Andric #elif defined(_MSC_VER) && defined(IS_X86_64)
9881ad6265SDimitry Andric   unsigned long index;
9981ad6265SDimitry Andric   _BitScanReverse64(&index, x);
10081ad6265SDimitry Andric   return index;
10181ad6265SDimitry Andric #elif defined(_MSC_VER) && defined(IS_X86_32)
10281ad6265SDimitry Andric   if(x >> 32) {
10381ad6265SDimitry Andric     unsigned long index;
10481ad6265SDimitry Andric     _BitScanReverse(&index, (unsigned long)(x >> 32));
10581ad6265SDimitry Andric     return 32 + index;
10681ad6265SDimitry Andric   } else {
10781ad6265SDimitry Andric     unsigned long index;
10881ad6265SDimitry Andric     _BitScanReverse(&index, (unsigned long)x);
10981ad6265SDimitry Andric     return index;
11081ad6265SDimitry Andric   }
11181ad6265SDimitry Andric #else
11281ad6265SDimitry Andric   unsigned int c = 0;
11381ad6265SDimitry Andric   if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
11481ad6265SDimitry Andric   if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
11581ad6265SDimitry Andric   if(x & 0x000000000000ff00ULL) { x >>=  8; c +=  8; }
11681ad6265SDimitry Andric   if(x & 0x00000000000000f0ULL) { x >>=  4; c +=  4; }
11781ad6265SDimitry Andric   if(x & 0x000000000000000cULL) { x >>=  2; c +=  2; }
11881ad6265SDimitry Andric   if(x & 0x0000000000000002ULL) {           c +=  1; }
11981ad6265SDimitry Andric   return c;
12081ad6265SDimitry Andric #endif
12181ad6265SDimitry Andric }
12281ad6265SDimitry Andric 
12381ad6265SDimitry Andric // Count the number of 1 bits.
popcnt(uint64_t x)12481ad6265SDimitry Andric INLINE unsigned int popcnt(uint64_t x) {
12581ad6265SDimitry Andric #if defined(__GNUC__) || defined(__clang__)
12681ad6265SDimitry Andric   return __builtin_popcountll(x);
12781ad6265SDimitry Andric #else
12881ad6265SDimitry Andric   unsigned int count = 0;
12981ad6265SDimitry Andric   while (x != 0) {
13081ad6265SDimitry Andric     count += 1;
13181ad6265SDimitry Andric     x &= x - 1;
13281ad6265SDimitry Andric   }
13381ad6265SDimitry Andric   return count;
13481ad6265SDimitry Andric #endif
13581ad6265SDimitry Andric }
13681ad6265SDimitry Andric 
13781ad6265SDimitry Andric // Largest power of two less than or equal to x. As a special case, returns 1
13881ad6265SDimitry Andric // when x is 0.
round_down_to_power_of_2(uint64_t x)13981ad6265SDimitry Andric INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
14081ad6265SDimitry Andric   return 1ULL << highest_one(x | 1);
14181ad6265SDimitry Andric }
14281ad6265SDimitry Andric 
counter_low(uint64_t counter)14381ad6265SDimitry Andric INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
14481ad6265SDimitry Andric 
counter_high(uint64_t counter)14581ad6265SDimitry Andric INLINE uint32_t counter_high(uint64_t counter) {
14681ad6265SDimitry Andric   return (uint32_t)(counter >> 32);
14781ad6265SDimitry Andric }
14881ad6265SDimitry Andric 
load32(const void * src)14981ad6265SDimitry Andric INLINE uint32_t load32(const void *src) {
15081ad6265SDimitry Andric   const uint8_t *p = (const uint8_t *)src;
15181ad6265SDimitry Andric   return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
15281ad6265SDimitry Andric          ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
15381ad6265SDimitry Andric }
15481ad6265SDimitry Andric 
load_key_words(const uint8_t key[BLAKE3_KEY_LEN],uint32_t key_words[8])15581ad6265SDimitry Andric INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
15681ad6265SDimitry Andric                            uint32_t key_words[8]) {
15781ad6265SDimitry Andric   key_words[0] = load32(&key[0 * 4]);
15881ad6265SDimitry Andric   key_words[1] = load32(&key[1 * 4]);
15981ad6265SDimitry Andric   key_words[2] = load32(&key[2 * 4]);
16081ad6265SDimitry Andric   key_words[3] = load32(&key[3 * 4]);
16181ad6265SDimitry Andric   key_words[4] = load32(&key[4 * 4]);
16281ad6265SDimitry Andric   key_words[5] = load32(&key[5 * 4]);
16381ad6265SDimitry Andric   key_words[6] = load32(&key[6 * 4]);
16481ad6265SDimitry Andric   key_words[7] = load32(&key[7 * 4]);
16581ad6265SDimitry Andric }
16681ad6265SDimitry Andric 
store32(void * dst,uint32_t w)16781ad6265SDimitry Andric INLINE void store32(void *dst, uint32_t w) {
16881ad6265SDimitry Andric   uint8_t *p = (uint8_t *)dst;
16981ad6265SDimitry Andric   p[0] = (uint8_t)(w >> 0);
17081ad6265SDimitry Andric   p[1] = (uint8_t)(w >> 8);
17181ad6265SDimitry Andric   p[2] = (uint8_t)(w >> 16);
17281ad6265SDimitry Andric   p[3] = (uint8_t)(w >> 24);
17381ad6265SDimitry Andric }
17481ad6265SDimitry Andric 
store_cv_words(uint8_t bytes_out[32],uint32_t cv_words[8])17581ad6265SDimitry Andric INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
17681ad6265SDimitry Andric   store32(&bytes_out[0 * 4], cv_words[0]);
17781ad6265SDimitry Andric   store32(&bytes_out[1 * 4], cv_words[1]);
17881ad6265SDimitry Andric   store32(&bytes_out[2 * 4], cv_words[2]);
17981ad6265SDimitry Andric   store32(&bytes_out[3 * 4], cv_words[3]);
18081ad6265SDimitry Andric   store32(&bytes_out[4 * 4], cv_words[4]);
18181ad6265SDimitry Andric   store32(&bytes_out[5 * 4], cv_words[5]);
18281ad6265SDimitry Andric   store32(&bytes_out[6 * 4], cv_words[6]);
18381ad6265SDimitry Andric   store32(&bytes_out[7 * 4], cv_words[7]);
18481ad6265SDimitry Andric }
18581ad6265SDimitry Andric 
18681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
18781ad6265SDimitry Andric void blake3_compress_in_place(uint32_t cv[8],
18881ad6265SDimitry Andric                               const uint8_t block[BLAKE3_BLOCK_LEN],
18981ad6265SDimitry Andric                               uint8_t block_len, uint64_t counter,
19081ad6265SDimitry Andric                               uint8_t flags);
19181ad6265SDimitry Andric 
19281ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
19381ad6265SDimitry Andric void blake3_compress_xof(const uint32_t cv[8],
19481ad6265SDimitry Andric                          const uint8_t block[BLAKE3_BLOCK_LEN],
19581ad6265SDimitry Andric                          uint8_t block_len, uint64_t counter, uint8_t flags,
19681ad6265SDimitry Andric                          uint8_t out[64]);
19781ad6265SDimitry Andric 
19881ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
19981ad6265SDimitry Andric void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
20081ad6265SDimitry Andric                       size_t blocks, const uint32_t key[8], uint64_t counter,
20181ad6265SDimitry Andric                       bool increment_counter, uint8_t flags,
20281ad6265SDimitry Andric                       uint8_t flags_start, uint8_t flags_end, uint8_t *out);
20381ad6265SDimitry Andric 
20481ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
20581ad6265SDimitry Andric size_t blake3_simd_degree(void);
20681ad6265SDimitry Andric 
20781ad6265SDimitry Andric 
20881ad6265SDimitry Andric // Declarations for implementation-specific functions.
20981ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
21081ad6265SDimitry Andric void blake3_compress_in_place_portable(uint32_t cv[8],
21181ad6265SDimitry Andric                                        const uint8_t block[BLAKE3_BLOCK_LEN],
21281ad6265SDimitry Andric                                        uint8_t block_len, uint64_t counter,
21381ad6265SDimitry Andric                                        uint8_t flags);
21481ad6265SDimitry Andric 
21581ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
21681ad6265SDimitry Andric void blake3_compress_xof_portable(const uint32_t cv[8],
21781ad6265SDimitry Andric                                   const uint8_t block[BLAKE3_BLOCK_LEN],
21881ad6265SDimitry Andric                                   uint8_t block_len, uint64_t counter,
21981ad6265SDimitry Andric                                   uint8_t flags, uint8_t out[64]);
22081ad6265SDimitry Andric 
22181ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
22281ad6265SDimitry Andric void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
22381ad6265SDimitry Andric                                size_t blocks, const uint32_t key[8],
22481ad6265SDimitry Andric                                uint64_t counter, bool increment_counter,
22581ad6265SDimitry Andric                                uint8_t flags, uint8_t flags_start,
22681ad6265SDimitry Andric                                uint8_t flags_end, uint8_t *out);
22781ad6265SDimitry Andric 
22881ad6265SDimitry Andric #if defined(IS_X86)
22981ad6265SDimitry Andric #if !defined(BLAKE3_NO_SSE2)
23081ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
23181ad6265SDimitry Andric void blake3_compress_in_place_sse2(uint32_t cv[8],
23281ad6265SDimitry Andric                                    const uint8_t block[BLAKE3_BLOCK_LEN],
23381ad6265SDimitry Andric                                    uint8_t block_len, uint64_t counter,
23481ad6265SDimitry Andric                                    uint8_t flags);
23581ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
23681ad6265SDimitry Andric void blake3_compress_xof_sse2(const uint32_t cv[8],
23781ad6265SDimitry Andric                               const uint8_t block[BLAKE3_BLOCK_LEN],
23881ad6265SDimitry Andric                               uint8_t block_len, uint64_t counter,
23981ad6265SDimitry Andric                               uint8_t flags, uint8_t out[64]);
24081ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
24181ad6265SDimitry Andric void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
24281ad6265SDimitry Andric                            size_t blocks, const uint32_t key[8],
24381ad6265SDimitry Andric                            uint64_t counter, bool increment_counter,
24481ad6265SDimitry Andric                            uint8_t flags, uint8_t flags_start,
24581ad6265SDimitry Andric                            uint8_t flags_end, uint8_t *out);
24681ad6265SDimitry Andric #endif
24781ad6265SDimitry Andric #if !defined(BLAKE3_NO_SSE41)
24881ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
24981ad6265SDimitry Andric void blake3_compress_in_place_sse41(uint32_t cv[8],
25081ad6265SDimitry Andric                                     const uint8_t block[BLAKE3_BLOCK_LEN],
25181ad6265SDimitry Andric                                     uint8_t block_len, uint64_t counter,
25281ad6265SDimitry Andric                                     uint8_t flags);
25381ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
25481ad6265SDimitry Andric void blake3_compress_xof_sse41(const uint32_t cv[8],
25581ad6265SDimitry Andric                                const uint8_t block[BLAKE3_BLOCK_LEN],
25681ad6265SDimitry Andric                                uint8_t block_len, uint64_t counter,
25781ad6265SDimitry Andric                                uint8_t flags, uint8_t out[64]);
25881ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
25981ad6265SDimitry Andric void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
26081ad6265SDimitry Andric                             size_t blocks, const uint32_t key[8],
26181ad6265SDimitry Andric                             uint64_t counter, bool increment_counter,
26281ad6265SDimitry Andric                             uint8_t flags, uint8_t flags_start,
26381ad6265SDimitry Andric                             uint8_t flags_end, uint8_t *out);
26481ad6265SDimitry Andric #endif
26581ad6265SDimitry Andric #if !defined(BLAKE3_NO_AVX2)
26681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
26781ad6265SDimitry Andric void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
26881ad6265SDimitry Andric                            size_t blocks, const uint32_t key[8],
26981ad6265SDimitry Andric                            uint64_t counter, bool increment_counter,
27081ad6265SDimitry Andric                            uint8_t flags, uint8_t flags_start,
27181ad6265SDimitry Andric                            uint8_t flags_end, uint8_t *out);
27281ad6265SDimitry Andric #endif
27381ad6265SDimitry Andric #if !defined(BLAKE3_NO_AVX512)
27481ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
27581ad6265SDimitry Andric void blake3_compress_in_place_avx512(uint32_t cv[8],
27681ad6265SDimitry Andric                                      const uint8_t block[BLAKE3_BLOCK_LEN],
27781ad6265SDimitry Andric                                      uint8_t block_len, uint64_t counter,
27881ad6265SDimitry Andric                                      uint8_t flags);
27981ad6265SDimitry Andric 
28081ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
28181ad6265SDimitry Andric void blake3_compress_xof_avx512(const uint32_t cv[8],
28281ad6265SDimitry Andric                                 const uint8_t block[BLAKE3_BLOCK_LEN],
28381ad6265SDimitry Andric                                 uint8_t block_len, uint64_t counter,
28481ad6265SDimitry Andric                                 uint8_t flags, uint8_t out[64]);
28581ad6265SDimitry Andric 
28681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
28781ad6265SDimitry Andric void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
28881ad6265SDimitry Andric                              size_t blocks, const uint32_t key[8],
28981ad6265SDimitry Andric                              uint64_t counter, bool increment_counter,
29081ad6265SDimitry Andric                              uint8_t flags, uint8_t flags_start,
29181ad6265SDimitry Andric                              uint8_t flags_end, uint8_t *out);
29281ad6265SDimitry Andric #endif
29381ad6265SDimitry Andric #endif
29481ad6265SDimitry Andric 
29581ad6265SDimitry Andric #if BLAKE3_USE_NEON == 1
29681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
29781ad6265SDimitry Andric void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
29881ad6265SDimitry Andric                            size_t blocks, const uint32_t key[8],
29981ad6265SDimitry Andric                            uint64_t counter, bool increment_counter,
30081ad6265SDimitry Andric                            uint8_t flags, uint8_t flags_start,
30181ad6265SDimitry Andric                            uint8_t flags_end, uint8_t *out);
30281ad6265SDimitry Andric #endif
30381ad6265SDimitry Andric 
30481ad6265SDimitry Andric 
30581ad6265SDimitry Andric #endif /* BLAKE3_IMPL_H */
306