1 #ifndef BLAKE3_IMPL_H
2 #define BLAKE3_IMPL_H
3
4 #include <assert.h>
5 #include <stdbool.h>
6 #include <stddef.h>
7 #include <stdint.h>
8 #include <string.h>
9
10 #include "llvm-c/blake3.h"
11 // For \p LLVM_LIBRARY_VISIBILITY
12 #include "llvm/Support/Compiler.h"
13
14 #include "llvm_blake3_prefix.h"
15
16 // internal flags
17 enum blake3_flags {
18 CHUNK_START = 1 << 0,
19 CHUNK_END = 1 << 1,
20 PARENT = 1 << 2,
21 ROOT = 1 << 3,
22 KEYED_HASH = 1 << 4,
23 DERIVE_KEY_CONTEXT = 1 << 5,
24 DERIVE_KEY_MATERIAL = 1 << 6,
25 };
26
27 // This C implementation tries to support recent versions of GCC, Clang, and
28 // MSVC.
29 #if defined(_MSC_VER)
30 #define INLINE static __forceinline
31 #else
32 #define INLINE static inline __attribute__((always_inline))
33 #endif
34
35 #if defined(__x86_64__) || defined(_M_X64)
36 #define IS_X86
37 #define IS_X86_64
38 #endif
39
40 #if defined(__i386__) || defined(_M_IX86)
41 #define IS_X86
42 #define IS_X86_32
43 #endif
44
45 #if defined(__aarch64__) || defined(_M_ARM64)
46 #define IS_AARCH64
47 #endif
48
49 #if defined(IS_X86)
50 #if defined(_MSC_VER)
51 #include <intrin.h>
52 #endif
53 #include <immintrin.h>
54 #endif
55
56 #if !defined(BLAKE3_USE_NEON)
57 // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness
58 #if defined(IS_AARCH64)
59 #define BLAKE3_USE_NEON 1
60 #else
61 #define BLAKE3_USE_NEON 0
62 #endif
63 #endif
64
65 #if defined(IS_X86)
66 #define MAX_SIMD_DEGREE 16
67 #elif BLAKE3_USE_NEON == 1
68 #define MAX_SIMD_DEGREE 4
69 #else
70 #define MAX_SIMD_DEGREE 1
71 #endif
72
73 // There are some places where we want a static size that's equal to the
74 // MAX_SIMD_DEGREE, but also at least 2.
75 #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)
76
77 static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,
78 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,
79 0x1F83D9ABUL, 0x5BE0CD19UL};
80
81 static const uint8_t MSG_SCHEDULE[7][16] = {
82 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
83 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},
84 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},
85 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},
86 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},
87 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},
88 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},
89 };
90
91 /* Find index of the highest set bit */
92 /* x is assumed to be nonzero. */
highest_one(uint64_t x)93 static unsigned int highest_one(uint64_t x) {
94 #if defined(__GNUC__) || defined(__clang__)
95 return 63 ^ __builtin_clzll(x);
96 #elif defined(_MSC_VER) && defined(IS_X86_64)
97 unsigned long index;
98 _BitScanReverse64(&index, x);
99 return index;
100 #elif defined(_MSC_VER) && defined(IS_X86_32)
101 if(x >> 32) {
102 unsigned long index;
103 _BitScanReverse(&index, (unsigned long)(x >> 32));
104 return 32 + index;
105 } else {
106 unsigned long index;
107 _BitScanReverse(&index, (unsigned long)x);
108 return index;
109 }
110 #else
111 unsigned int c = 0;
112 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
113 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
114 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
115 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
116 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }
117 if(x & 0x0000000000000002ULL) { c += 1; }
118 return c;
119 #endif
120 }
121
122 // Count the number of 1 bits.
popcnt(uint64_t x)123 INLINE unsigned int popcnt(uint64_t x) {
124 #if defined(__GNUC__) || defined(__clang__)
125 return __builtin_popcountll(x);
126 #else
127 unsigned int count = 0;
128 while (x != 0) {
129 count += 1;
130 x &= x - 1;
131 }
132 return count;
133 #endif
134 }
135
136 // Largest power of two less than or equal to x. As a special case, returns 1
137 // when x is 0.
round_down_to_power_of_2(uint64_t x)138 INLINE uint64_t round_down_to_power_of_2(uint64_t x) {
139 return 1ULL << highest_one(x | 1);
140 }
141
counter_low(uint64_t counter)142 INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }
143
counter_high(uint64_t counter)144 INLINE uint32_t counter_high(uint64_t counter) {
145 return (uint32_t)(counter >> 32);
146 }
147
load32(const void * src)148 INLINE uint32_t load32(const void *src) {
149 const uint8_t *p = (const uint8_t *)src;
150 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
151 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
152 }
153
load_key_words(const uint8_t key[BLAKE3_KEY_LEN],uint32_t key_words[8])154 INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],
155 uint32_t key_words[8]) {
156 key_words[0] = load32(&key[0 * 4]);
157 key_words[1] = load32(&key[1 * 4]);
158 key_words[2] = load32(&key[2 * 4]);
159 key_words[3] = load32(&key[3 * 4]);
160 key_words[4] = load32(&key[4 * 4]);
161 key_words[5] = load32(&key[5 * 4]);
162 key_words[6] = load32(&key[6 * 4]);
163 key_words[7] = load32(&key[7 * 4]);
164 }
165
store32(void * dst,uint32_t w)166 INLINE void store32(void *dst, uint32_t w) {
167 uint8_t *p = (uint8_t *)dst;
168 p[0] = (uint8_t)(w >> 0);
169 p[1] = (uint8_t)(w >> 8);
170 p[2] = (uint8_t)(w >> 16);
171 p[3] = (uint8_t)(w >> 24);
172 }
173
store_cv_words(uint8_t bytes_out[32],uint32_t cv_words[8])174 INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {
175 store32(&bytes_out[0 * 4], cv_words[0]);
176 store32(&bytes_out[1 * 4], cv_words[1]);
177 store32(&bytes_out[2 * 4], cv_words[2]);
178 store32(&bytes_out[3 * 4], cv_words[3]);
179 store32(&bytes_out[4 * 4], cv_words[4]);
180 store32(&bytes_out[5 * 4], cv_words[5]);
181 store32(&bytes_out[6 * 4], cv_words[6]);
182 store32(&bytes_out[7 * 4], cv_words[7]);
183 }
184
185 LLVM_LIBRARY_VISIBILITY
186 void blake3_compress_in_place(uint32_t cv[8],
187 const uint8_t block[BLAKE3_BLOCK_LEN],
188 uint8_t block_len, uint64_t counter,
189 uint8_t flags);
190
191 LLVM_LIBRARY_VISIBILITY
192 void blake3_compress_xof(const uint32_t cv[8],
193 const uint8_t block[BLAKE3_BLOCK_LEN],
194 uint8_t block_len, uint64_t counter, uint8_t flags,
195 uint8_t out[64]);
196
197 LLVM_LIBRARY_VISIBILITY
198 void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,
199 size_t blocks, const uint32_t key[8], uint64_t counter,
200 bool increment_counter, uint8_t flags,
201 uint8_t flags_start, uint8_t flags_end, uint8_t *out);
202
203 LLVM_LIBRARY_VISIBILITY
204 size_t blake3_simd_degree(void);
205
206
207 // Declarations for implementation-specific functions.
208 LLVM_LIBRARY_VISIBILITY
209 void blake3_compress_in_place_portable(uint32_t cv[8],
210 const uint8_t block[BLAKE3_BLOCK_LEN],
211 uint8_t block_len, uint64_t counter,
212 uint8_t flags);
213
214 LLVM_LIBRARY_VISIBILITY
215 void blake3_compress_xof_portable(const uint32_t cv[8],
216 const uint8_t block[BLAKE3_BLOCK_LEN],
217 uint8_t block_len, uint64_t counter,
218 uint8_t flags, uint8_t out[64]);
219
220 LLVM_LIBRARY_VISIBILITY
221 void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
222 size_t blocks, const uint32_t key[8],
223 uint64_t counter, bool increment_counter,
224 uint8_t flags, uint8_t flags_start,
225 uint8_t flags_end, uint8_t *out);
226
227 #if defined(IS_X86)
228 #if !defined(BLAKE3_NO_SSE2)
229 LLVM_LIBRARY_VISIBILITY
230 void blake3_compress_in_place_sse2(uint32_t cv[8],
231 const uint8_t block[BLAKE3_BLOCK_LEN],
232 uint8_t block_len, uint64_t counter,
233 uint8_t flags);
234 LLVM_LIBRARY_VISIBILITY
235 void blake3_compress_xof_sse2(const uint32_t cv[8],
236 const uint8_t block[BLAKE3_BLOCK_LEN],
237 uint8_t block_len, uint64_t counter,
238 uint8_t flags, uint8_t out[64]);
239 LLVM_LIBRARY_VISIBILITY
240 void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
241 size_t blocks, const uint32_t key[8],
242 uint64_t counter, bool increment_counter,
243 uint8_t flags, uint8_t flags_start,
244 uint8_t flags_end, uint8_t *out);
245 #endif
246 #if !defined(BLAKE3_NO_SSE41)
247 LLVM_LIBRARY_VISIBILITY
248 void blake3_compress_in_place_sse41(uint32_t cv[8],
249 const uint8_t block[BLAKE3_BLOCK_LEN],
250 uint8_t block_len, uint64_t counter,
251 uint8_t flags);
252 LLVM_LIBRARY_VISIBILITY
253 void blake3_compress_xof_sse41(const uint32_t cv[8],
254 const uint8_t block[BLAKE3_BLOCK_LEN],
255 uint8_t block_len, uint64_t counter,
256 uint8_t flags, uint8_t out[64]);
257 LLVM_LIBRARY_VISIBILITY
258 void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
259 size_t blocks, const uint32_t key[8],
260 uint64_t counter, bool increment_counter,
261 uint8_t flags, uint8_t flags_start,
262 uint8_t flags_end, uint8_t *out);
263 #endif
264 #if !defined(BLAKE3_NO_AVX2)
265 LLVM_LIBRARY_VISIBILITY
266 void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
267 size_t blocks, const uint32_t key[8],
268 uint64_t counter, bool increment_counter,
269 uint8_t flags, uint8_t flags_start,
270 uint8_t flags_end, uint8_t *out);
271 #endif
272 #if !defined(BLAKE3_NO_AVX512)
273 LLVM_LIBRARY_VISIBILITY
274 void blake3_compress_in_place_avx512(uint32_t cv[8],
275 const uint8_t block[BLAKE3_BLOCK_LEN],
276 uint8_t block_len, uint64_t counter,
277 uint8_t flags);
278
279 LLVM_LIBRARY_VISIBILITY
280 void blake3_compress_xof_avx512(const uint32_t cv[8],
281 const uint8_t block[BLAKE3_BLOCK_LEN],
282 uint8_t block_len, uint64_t counter,
283 uint8_t flags, uint8_t out[64]);
284
285 LLVM_LIBRARY_VISIBILITY
286 void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
287 size_t blocks, const uint32_t key[8],
288 uint64_t counter, bool increment_counter,
289 uint8_t flags, uint8_t flags_start,
290 uint8_t flags_end, uint8_t *out);
291 #endif
292 #endif
293
294 #if BLAKE3_USE_NEON == 1
295 LLVM_LIBRARY_VISIBILITY
296 void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
297 size_t blocks, const uint32_t key[8],
298 uint64_t counter, bool increment_counter,
299 uint8_t flags, uint8_t flags_start,
300 uint8_t flags_end, uint8_t *out);
301 #endif
302
303
304 #endif /* BLAKE3_IMPL_H */
305