181ad6265SDimitry Andric #ifndef BLAKE3_IMPL_H
281ad6265SDimitry Andric #define BLAKE3_IMPL_H
381ad6265SDimitry Andric
481ad6265SDimitry Andric #include <assert.h>
581ad6265SDimitry Andric #include <stdbool.h>
681ad6265SDimitry Andric #include <stddef.h>
781ad6265SDimitry Andric #include <stdint.h>
881ad6265SDimitry Andric #include <string.h>
981ad6265SDimitry Andric
1081ad6265SDimitry Andric #include "llvm-c/blake3.h"
1181ad6265SDimitry Andric // For \p LLVM_LIBRARY_VISIBILITY
1281ad6265SDimitry Andric #include "llvm/Support/Compiler.h"
1381ad6265SDimitry Andric
141ac55f4cSDimitry Andric #include "llvm_blake3_prefix.h"
1581ad6265SDimitry Andric
1681ad6265SDimitry Andric // internal flags
1781ad6265SDimitry Andric enum blake3_flags {
1881ad6265SDimitry Andric CHUNK_START = 1 << 0,
1981ad6265SDimitry Andric CHUNK_END = 1 << 1,
2081ad6265SDimitry Andric PARENT = 1 << 2,
2181ad6265SDimitry Andric ROOT = 1 << 3,
2281ad6265SDimitry Andric KEYED_HASH = 1 << 4,
2381ad6265SDimitry Andric DERIVE_KEY_CONTEXT = 1 << 5,
2481ad6265SDimitry Andric DERIVE_KEY_MATERIAL = 1 << 6,
2581ad6265SDimitry Andric };
2681ad6265SDimitry Andric
2781ad6265SDimitry Andric // This C implementation tries to support recent versions of GCC, Clang, and
2881ad6265SDimitry Andric // MSVC.
2981ad6265SDimitry Andric #if defined(_MSC_VER)
3081ad6265SDimitry Andric #define INLINE static __forceinline
3181ad6265SDimitry Andric #else
3281ad6265SDimitry Andric #define INLINE static inline __attribute__((always_inline))
3381ad6265SDimitry Andric #endif
3481ad6265SDimitry Andric
3581ad6265SDimitry Andric #if defined(__x86_64__) || defined(_M_X64)
3681ad6265SDimitry Andric #define IS_X86
3781ad6265SDimitry Andric #define IS_X86_64
3881ad6265SDimitry Andric #endif
3981ad6265SDimitry Andric
4081ad6265SDimitry Andric #if defined(__i386__) || defined(_M_IX86)
4181ad6265SDimitry Andric #define IS_X86
4281ad6265SDimitry Andric #define IS_X86_32
4381ad6265SDimitry Andric #endif
4481ad6265SDimitry Andric
4581ad6265SDimitry Andric #if defined(__aarch64__) || defined(_M_ARM64)
4681ad6265SDimitry Andric #define IS_AARCH64
4781ad6265SDimitry Andric #endif
4881ad6265SDimitry Andric
4981ad6265SDimitry Andric #if defined(IS_X86)
5081ad6265SDimitry Andric #if defined(_MSC_VER)
5181ad6265SDimitry Andric #include <intrin.h>
5281ad6265SDimitry Andric #endif
5381ad6265SDimitry Andric #include <immintrin.h>
5481ad6265SDimitry Andric #endif
5581ad6265SDimitry Andric
5681ad6265SDimitry Andric #if !defined(BLAKE3_USE_NEON)
57*5f757f3fSDimitry Andric // If BLAKE3_USE_NEON not manually set, autodetect based on
58*5f757f3fSDimitry Andric // AArch64ness and endianness.
59*5f757f3fSDimitry Andric #if defined(IS_AARCH64) && !defined(__ARM_BIG_ENDIAN)
6081ad6265SDimitry Andric #define BLAKE3_USE_NEON 1
6181ad6265SDimitry Andric #else
6281ad6265SDimitry Andric #define BLAKE3_USE_NEON 0
6381ad6265SDimitry Andric #endif
6481ad6265SDimitry Andric #endif
6581ad6265SDimitry Andric
6681ad6265SDimitry Andric #if defined(IS_X86)
6781ad6265SDimitry Andric #define MAX_SIMD_DEGREE 16
6881ad6265SDimitry Andric #elif BLAKE3_USE_NEON == 1
6981ad6265SDimitry Andric #define MAX_SIMD_DEGREE 4
7081ad6265SDimitry Andric #else
7181ad6265SDimitry Andric #define MAX_SIMD_DEGREE 1
7281ad6265SDimitry Andric #endif
7381ad6265SDimitry Andric
7481ad6265SDimitry Andric // There are some places where we want a static size that's equal to the
7581ad6265SDimitry Andric // MAX_SIMD_DEGREE, but also at least 2.
7681ad6265SDimitry Andric #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
7781ad6265SDimitry Andric
7881ad6265SDimitry Andric static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
7981ad6265SDimitry Andric 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
8081ad6265SDimitry Andric 0x1F83D9ABUL, 0x5BE0CD19UL};
8181ad6265SDimitry Andric
8281ad6265SDimitry Andric static const uint8_t MSG_SCHEDULE[7][16] = {
8381ad6265SDimitry Andric {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
8481ad6265SDimitry Andric {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
8581ad6265SDimitry Andric {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
8681ad6265SDimitry Andric {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
8781ad6265SDimitry Andric {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
8881ad6265SDimitry Andric {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
8981ad6265SDimitry Andric {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
9081ad6265SDimitry Andric };
9181ad6265SDimitry Andric
9281ad6265SDimitry Andric /* Find index of the highest set bit */
9381ad6265SDimitry Andric /* x is assumed to be nonzero. */
highest_one(uint64_t x)9481ad6265SDimitry Andric static unsigned int highest_one(uint64_t x) {
9581ad6265SDimitry Andric #if defined(__GNUC__) || defined(__clang__)
9681ad6265SDimitry Andric return 63 ^ __builtin_clzll(x);
9781ad6265SDimitry Andric #elif defined(_MSC_VER) && defined(IS_X86_64)
9881ad6265SDimitry Andric unsigned long index;
9981ad6265SDimitry Andric _BitScanReverse64(&index, x);
10081ad6265SDimitry Andric return index;
10181ad6265SDimitry Andric #elif defined(_MSC_VER) && defined(IS_X86_32)
10281ad6265SDimitry Andric if(x >> 32) {
10381ad6265SDimitry Andric unsigned long index;
10481ad6265SDimitry Andric _BitScanReverse(&index, (unsigned long)(x >> 32));
10581ad6265SDimitry Andric return 32 + index;
10681ad6265SDimitry Andric } else {
10781ad6265SDimitry Andric unsigned long index;
10881ad6265SDimitry Andric _BitScanReverse(&index, (unsigned long)x);
10981ad6265SDimitry Andric return index;
11081ad6265SDimitry Andric }
11181ad6265SDimitry Andric #else
11281ad6265SDimitry Andric unsigned int c = 0;
11381ad6265SDimitry Andric if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
11481ad6265SDimitry Andric if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
11581ad6265SDimitry Andric if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
11681ad6265SDimitry Andric if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
11781ad6265SDimitry Andric if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }
11881ad6265SDimitry Andric if(x & 0x0000000000000002ULL) { c += 1; }
11981ad6265SDimitry Andric return c;
12081ad6265SDimitry Andric #endif
12181ad6265SDimitry Andric }
12281ad6265SDimitry Andric
12381ad6265SDimitry Andric // Count the number of 1 bits.
popcnt(uint64_t x)12481ad6265SDimitry Andric INLINE unsigned int popcnt(uint64_t x) {
12581ad6265SDimitry Andric #if defined(__GNUC__) || defined(__clang__)
12681ad6265SDimitry Andric return __builtin_popcountll(x);
12781ad6265SDimitry Andric #else
12881ad6265SDimitry Andric unsigned int count = 0;
12981ad6265SDimitry Andric while (x != 0) {
13081ad6265SDimitry Andric count += 1;
13181ad6265SDimitry Andric x &= x - 1;
13281ad6265SDimitry Andric }
13381ad6265SDimitry Andric return count;
13481ad6265SDimitry Andric #endif
13581ad6265SDimitry Andric }
13681ad6265SDimitry Andric
13781ad6265SDimitry Andric // Largest power of two less than or equal to x. As a special case, returns 1
13881ad6265SDimitry Andric // when x is 0.
round_down_to_power_of_2(uint64_t x)13981ad6265SDimitry Andric INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
14081ad6265SDimitry Andric return 1ULL << highest_one(x | 1);
14181ad6265SDimitry Andric }
14281ad6265SDimitry Andric
counter_low(uint64_t counter)14381ad6265SDimitry Andric INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
14481ad6265SDimitry Andric
counter_high(uint64_t counter)14581ad6265SDimitry Andric INLINE uint32_t counter_high(uint64_t counter) {
14681ad6265SDimitry Andric return (uint32_t)(counter >> 32);
14781ad6265SDimitry Andric }
14881ad6265SDimitry Andric
load32(const void * src)14981ad6265SDimitry Andric INLINE uint32_t load32(const void *src) {
15081ad6265SDimitry Andric const uint8_t *p = (const uint8_t *)src;
15181ad6265SDimitry Andric return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
15281ad6265SDimitry Andric ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
15381ad6265SDimitry Andric }
15481ad6265SDimitry Andric
load_key_words(const uint8_t key[BLAKE3_KEY_LEN],uint32_t key_words[8])15581ad6265SDimitry Andric INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
15681ad6265SDimitry Andric uint32_t key_words[8]) {
15781ad6265SDimitry Andric key_words[0] = load32(&key[0 * 4]);
15881ad6265SDimitry Andric key_words[1] = load32(&key[1 * 4]);
15981ad6265SDimitry Andric key_words[2] = load32(&key[2 * 4]);
16081ad6265SDimitry Andric key_words[3] = load32(&key[3 * 4]);
16181ad6265SDimitry Andric key_words[4] = load32(&key[4 * 4]);
16281ad6265SDimitry Andric key_words[5] = load32(&key[5 * 4]);
16381ad6265SDimitry Andric key_words[6] = load32(&key[6 * 4]);
16481ad6265SDimitry Andric key_words[7] = load32(&key[7 * 4]);
16581ad6265SDimitry Andric }
16681ad6265SDimitry Andric
store32(void * dst,uint32_t w)16781ad6265SDimitry Andric INLINE void store32(void *dst, uint32_t w) {
16881ad6265SDimitry Andric uint8_t *p = (uint8_t *)dst;
16981ad6265SDimitry Andric p[0] = (uint8_t)(w >> 0);
17081ad6265SDimitry Andric p[1] = (uint8_t)(w >> 8);
17181ad6265SDimitry Andric p[2] = (uint8_t)(w >> 16);
17281ad6265SDimitry Andric p[3] = (uint8_t)(w >> 24);
17381ad6265SDimitry Andric }
17481ad6265SDimitry Andric
store_cv_words(uint8_t bytes_out[32],uint32_t cv_words[8])17581ad6265SDimitry Andric INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
17681ad6265SDimitry Andric store32(&bytes_out[0 * 4], cv_words[0]);
17781ad6265SDimitry Andric store32(&bytes_out[1 * 4], cv_words[1]);
17881ad6265SDimitry Andric store32(&bytes_out[2 * 4], cv_words[2]);
17981ad6265SDimitry Andric store32(&bytes_out[3 * 4], cv_words[3]);
18081ad6265SDimitry Andric store32(&bytes_out[4 * 4], cv_words[4]);
18181ad6265SDimitry Andric store32(&bytes_out[5 * 4], cv_words[5]);
18281ad6265SDimitry Andric store32(&bytes_out[6 * 4], cv_words[6]);
18381ad6265SDimitry Andric store32(&bytes_out[7 * 4], cv_words[7]);
18481ad6265SDimitry Andric }
18581ad6265SDimitry Andric
18681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
18781ad6265SDimitry Andric void blake3_compress_in_place(uint32_t cv[8],
18881ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
18981ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
19081ad6265SDimitry Andric uint8_t flags);
19181ad6265SDimitry Andric
19281ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
19381ad6265SDimitry Andric void blake3_compress_xof(const uint32_t cv[8],
19481ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
19581ad6265SDimitry Andric uint8_t block_len, uint64_t counter, uint8_t flags,
19681ad6265SDimitry Andric uint8_t out[64]);
19781ad6265SDimitry Andric
19881ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
19981ad6265SDimitry Andric void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
20081ad6265SDimitry Andric size_t blocks, const uint32_t key[8], uint64_t counter,
20181ad6265SDimitry Andric bool increment_counter, uint8_t flags,
20281ad6265SDimitry Andric uint8_t flags_start, uint8_t flags_end, uint8_t *out);
20381ad6265SDimitry Andric
20481ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
20581ad6265SDimitry Andric size_t blake3_simd_degree(void);
20681ad6265SDimitry Andric
20781ad6265SDimitry Andric
20881ad6265SDimitry Andric // Declarations for implementation-specific functions.
20981ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
21081ad6265SDimitry Andric void blake3_compress_in_place_portable(uint32_t cv[8],
21181ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
21281ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
21381ad6265SDimitry Andric uint8_t flags);
21481ad6265SDimitry Andric
21581ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
21681ad6265SDimitry Andric void blake3_compress_xof_portable(const uint32_t cv[8],
21781ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
21881ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
21981ad6265SDimitry Andric uint8_t flags, uint8_t out[64]);
22081ad6265SDimitry Andric
22181ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
22281ad6265SDimitry Andric void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
22381ad6265SDimitry Andric size_t blocks, const uint32_t key[8],
22481ad6265SDimitry Andric uint64_t counter, bool increment_counter,
22581ad6265SDimitry Andric uint8_t flags, uint8_t flags_start,
22681ad6265SDimitry Andric uint8_t flags_end, uint8_t *out);
22781ad6265SDimitry Andric
22881ad6265SDimitry Andric #if defined(IS_X86)
22981ad6265SDimitry Andric #if !defined(BLAKE3_NO_SSE2)
23081ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
23181ad6265SDimitry Andric void blake3_compress_in_place_sse2(uint32_t cv[8],
23281ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
23381ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
23481ad6265SDimitry Andric uint8_t flags);
23581ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
23681ad6265SDimitry Andric void blake3_compress_xof_sse2(const uint32_t cv[8],
23781ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
23881ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
23981ad6265SDimitry Andric uint8_t flags, uint8_t out[64]);
24081ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
24181ad6265SDimitry Andric void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
24281ad6265SDimitry Andric size_t blocks, const uint32_t key[8],
24381ad6265SDimitry Andric uint64_t counter, bool increment_counter,
24481ad6265SDimitry Andric uint8_t flags, uint8_t flags_start,
24581ad6265SDimitry Andric uint8_t flags_end, uint8_t *out);
24681ad6265SDimitry Andric #endif
24781ad6265SDimitry Andric #if !defined(BLAKE3_NO_SSE41)
24881ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
24981ad6265SDimitry Andric void blake3_compress_in_place_sse41(uint32_t cv[8],
25081ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
25181ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
25281ad6265SDimitry Andric uint8_t flags);
25381ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
25481ad6265SDimitry Andric void blake3_compress_xof_sse41(const uint32_t cv[8],
25581ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
25681ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
25781ad6265SDimitry Andric uint8_t flags, uint8_t out[64]);
25881ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
25981ad6265SDimitry Andric void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
26081ad6265SDimitry Andric size_t blocks, const uint32_t key[8],
26181ad6265SDimitry Andric uint64_t counter, bool increment_counter,
26281ad6265SDimitry Andric uint8_t flags, uint8_t flags_start,
26381ad6265SDimitry Andric uint8_t flags_end, uint8_t *out);
26481ad6265SDimitry Andric #endif
26581ad6265SDimitry Andric #if !defined(BLAKE3_NO_AVX2)
26681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
26781ad6265SDimitry Andric void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
26881ad6265SDimitry Andric size_t blocks, const uint32_t key[8],
26981ad6265SDimitry Andric uint64_t counter, bool increment_counter,
27081ad6265SDimitry Andric uint8_t flags, uint8_t flags_start,
27181ad6265SDimitry Andric uint8_t flags_end, uint8_t *out);
27281ad6265SDimitry Andric #endif
27381ad6265SDimitry Andric #if !defined(BLAKE3_NO_AVX512)
27481ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
27581ad6265SDimitry Andric void blake3_compress_in_place_avx512(uint32_t cv[8],
27681ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
27781ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
27881ad6265SDimitry Andric uint8_t flags);
27981ad6265SDimitry Andric
28081ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
28181ad6265SDimitry Andric void blake3_compress_xof_avx512(const uint32_t cv[8],
28281ad6265SDimitry Andric const uint8_t block[BLAKE3_BLOCK_LEN],
28381ad6265SDimitry Andric uint8_t block_len, uint64_t counter,
28481ad6265SDimitry Andric uint8_t flags, uint8_t out[64]);
28581ad6265SDimitry Andric
28681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
28781ad6265SDimitry Andric void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
28881ad6265SDimitry Andric size_t blocks, const uint32_t key[8],
28981ad6265SDimitry Andric uint64_t counter, bool increment_counter,
29081ad6265SDimitry Andric uint8_t flags, uint8_t flags_start,
29181ad6265SDimitry Andric uint8_t flags_end, uint8_t *out);
29281ad6265SDimitry Andric #endif
29381ad6265SDimitry Andric #endif
29481ad6265SDimitry Andric
29581ad6265SDimitry Andric #if BLAKE3_USE_NEON == 1
29681ad6265SDimitry Andric LLVM_LIBRARY_VISIBILITY
29781ad6265SDimitry Andric void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
29881ad6265SDimitry Andric size_t blocks, const uint32_t key[8],
29981ad6265SDimitry Andric uint64_t counter, bool increment_counter,
30081ad6265SDimitry Andric uint8_t flags, uint8_t flags_start,
30181ad6265SDimitry Andric uint8_t flags_end, uint8_t *out);
30281ad6265SDimitry Andric #endif
30381ad6265SDimitry Andric
30481ad6265SDimitry Andric
30581ad6265SDimitry Andric #endif /* BLAKE3_IMPL_H */
306