1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #include <string.h>
31 #include "sha256_mb_internal.h"
32 #include "memcpy_inline.h"
33 #include "endian_helper.h"
34
35 #ifdef _MSC_VER
36 #include <intrin.h>
37 #define inline __inline
38 #endif
39
40 #if (__GNUC__ >= 11)
41 #define OPT_FIX __attribute__((noipa))
42 #else
43 #define OPT_FIX
44 #endif
45
46 #define ror32(x, r) (((x) >> (r)) ^ ((x) << (32 - (r))))
47
48 #define W(x) w[(x) & 15]
49
50 #define S0(w) (ror32(w, 7) ^ ror32(w, 18) ^ (w >> 3))
51 #define S1(w) (ror32(w, 17) ^ ror32(w, 19) ^ (w >> 10))
52
53 #define s0(a) (ror32(a, 2) ^ ror32(a, 13) ^ ror32(a, 22))
54 #define s1(e) (ror32(e, 6) ^ ror32(e, 11) ^ ror32(e, 25))
55 #define maj(a, b, c) ((a & b) ^ (a & c) ^ (b & c))
56 #define ch(e, f, g) ((e & f) ^ (g & ~e))
57
58 #define step(i, a, b, c, d, e, f, g, h, k) \
59 if (i < 16) \
60 W(i) = to_be32(ww[i]); \
61 else \
62 W(i) = W(i - 16) + S0(W(i - 15)) + W(i - 7) + S1(W(i - 2)); \
63 t2 = s0(a) + maj(a, b, c); \
64 t1 = h + s1(e) + ch(e, f, g) + k + W(i); \
65 d += t1; \
66 h = t1 + t2;
67
68 static void
69 sha256_init(ISAL_SHA256_HASH_CTX *ctx, const void *buffer, uint32_t len);
70 static void
71 sha256_update(ISAL_SHA256_HASH_CTX *ctx, const void *buffer, uint32_t len);
72 static void
73 sha256_final(ISAL_SHA256_HASH_CTX *ctx);
74 static void OPT_FIX
75 sha256_single(const void *data, uint32_t digest[]);
76 static inline void
77 hash_init_digest(ISAL_SHA256_WORD_T *digest);
78
79 void
_sha256_ctx_mgr_init_base(ISAL_SHA256_HASH_CTX_MGR * mgr)80 _sha256_ctx_mgr_init_base(ISAL_SHA256_HASH_CTX_MGR *mgr)
81 {
82 }
83
84 ISAL_SHA256_HASH_CTX *
_sha256_ctx_mgr_submit_base(ISAL_SHA256_HASH_CTX_MGR * mgr,ISAL_SHA256_HASH_CTX * ctx,const void * buffer,uint32_t len,ISAL_HASH_CTX_FLAG flags)85 _sha256_ctx_mgr_submit_base(ISAL_SHA256_HASH_CTX_MGR *mgr, ISAL_SHA256_HASH_CTX *ctx,
86 const void *buffer, uint32_t len, ISAL_HASH_CTX_FLAG flags)
87 {
88
89 if (flags & (~ISAL_HASH_ENTIRE)) {
90 // User should not pass anything other than FIRST, UPDATE, or LAST
91 ctx->error = ISAL_HASH_CTX_ERROR_INVALID_FLAGS;
92 return ctx;
93 }
94
95 if ((ctx->status & ISAL_HASH_CTX_STS_PROCESSING) && (flags == ISAL_HASH_ENTIRE)) {
96 // Cannot submit a new entire job to a currently processing job.
97 ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_PROCESSING;
98 return ctx;
99 }
100
101 if ((ctx->status & ISAL_HASH_CTX_STS_COMPLETE) && !(flags & ISAL_HASH_FIRST)) {
102 // Cannot update a finished job.
103 ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_COMPLETED;
104 return ctx;
105 }
106
107 if (flags == ISAL_HASH_FIRST) {
108
109 sha256_init(ctx, buffer, len);
110 sha256_update(ctx, buffer, len);
111 }
112
113 if (flags == ISAL_HASH_UPDATE) {
114 sha256_update(ctx, buffer, len);
115 }
116
117 if (flags == ISAL_HASH_LAST) {
118 sha256_update(ctx, buffer, len);
119 sha256_final(ctx);
120 }
121
122 if (flags == ISAL_HASH_ENTIRE) {
123 sha256_init(ctx, buffer, len);
124 sha256_update(ctx, buffer, len);
125 sha256_final(ctx);
126 }
127
128 return ctx;
129 }
130
131 ISAL_SHA256_HASH_CTX *
_sha256_ctx_mgr_flush_base(ISAL_SHA256_HASH_CTX_MGR * mgr)132 _sha256_ctx_mgr_flush_base(ISAL_SHA256_HASH_CTX_MGR *mgr)
133 {
134 return NULL;
135 }
136
137 static void
sha256_init(ISAL_SHA256_HASH_CTX * ctx,const void * buffer,uint32_t len)138 sha256_init(ISAL_SHA256_HASH_CTX *ctx, const void *buffer, uint32_t len)
139 {
140 // Init digest
141 hash_init_digest(ctx->job.result_digest);
142
143 // Reset byte counter
144 ctx->total_length = 0;
145
146 // Clear extra blocks
147 ctx->partial_block_buffer_length = 0;
148
149 // If we made it here, there were no errors during this call to submit
150 ctx->error = ISAL_HASH_CTX_ERROR_NONE;
151
152 // Mark it as processing
153 ctx->status = ISAL_HASH_CTX_STS_PROCESSING;
154 }
155
156 static void
sha256_update(ISAL_SHA256_HASH_CTX * ctx,const void * buffer,uint32_t len)157 sha256_update(ISAL_SHA256_HASH_CTX *ctx, const void *buffer, uint32_t len)
158 {
159 uint32_t remain_len = len;
160 uint32_t *digest = ctx->job.result_digest;
161
162 // Advance byte counter
163 ctx->total_length += len;
164
165 // If there is anything currently buffered in the extra blocks, append to it until it
166 // contains a whole block. Or if the user's buffer contains less than a whole block, append
167 // as much as possible to the extra block.
168 if ((ctx->partial_block_buffer_length) | (remain_len < ISAL_SHA256_BLOCK_SIZE)) {
169 // Compute how many bytes to copy from user buffer into extra block
170 uint32_t copy_len = ISAL_SHA256_BLOCK_SIZE - ctx->partial_block_buffer_length;
171 if (remain_len < copy_len) {
172 copy_len = remain_len;
173 }
174
175 if (copy_len) {
176 // Copy and update relevant pointers and counters
177 memcpy_fixedlen(
178 &ctx->partial_block_buffer[ctx->partial_block_buffer_length],
179 buffer, copy_len);
180
181 ctx->partial_block_buffer_length += copy_len;
182 remain_len -= copy_len;
183 buffer = (void *) ((uint8_t *) buffer + copy_len);
184 }
185 // The extra block should never contain more than 1 block here
186 assert(ctx->partial_block_buffer_length <= ISAL_SHA256_BLOCK_SIZE);
187
188 // If the extra block buffer contains exactly 1 block, it can be hashed.
189 if (ctx->partial_block_buffer_length >= ISAL_SHA256_BLOCK_SIZE) {
190 ctx->partial_block_buffer_length = 0;
191 sha256_single(ctx->partial_block_buffer, digest);
192 }
193 }
194 // If the extra blocks are empty, begin hashing what remains in the user's buffer.
195 if (ctx->partial_block_buffer_length == 0) {
196 while (remain_len >= ISAL_SHA256_BLOCK_SIZE) {
197 sha256_single(buffer, digest);
198 buffer = (void *) ((uint8_t *) buffer + ISAL_SHA256_BLOCK_SIZE);
199 remain_len -= ISAL_SHA256_BLOCK_SIZE;
200 }
201 }
202
203 if (remain_len > 0) {
204 memcpy_fixedlen(&ctx->partial_block_buffer, buffer, remain_len);
205 ctx->partial_block_buffer_length = remain_len;
206 }
207
208 ctx->status = ISAL_HASH_CTX_STS_IDLE;
209 return;
210 }
211
212 static void
sha256_final(ISAL_SHA256_HASH_CTX * ctx)213 sha256_final(ISAL_SHA256_HASH_CTX *ctx)
214 {
215
216 const void *buffer = ctx->partial_block_buffer;
217 uint32_t i = ctx->partial_block_buffer_length;
218 uint8_t buf[2 * ISAL_SHA256_BLOCK_SIZE];
219 uint32_t *digest = ctx->job.result_digest;
220
221 memcpy(buf, buffer, i);
222 buf[i++] = 0x80;
223 for (uint32_t j = i; j < (2 * ISAL_SHA256_BLOCK_SIZE); j++) {
224 buf[j] = 0;
225 }
226
227 if (i > ISAL_SHA256_BLOCK_SIZE - ISAL_SHA256_PADLENGTHFIELD_SIZE) {
228 i = 2 * ISAL_SHA256_BLOCK_SIZE;
229 } else {
230 i = ISAL_SHA256_BLOCK_SIZE;
231 }
232
233 *(uint64_t *) (buf + i - 8) = to_be64((uint64_t) ctx->total_length * 8);
234
235 sha256_single(buf, digest);
236 if (i == 2 * ISAL_SHA256_BLOCK_SIZE) {
237 sha256_single(buf + ISAL_SHA256_BLOCK_SIZE, digest);
238 }
239
240 ctx->status = ISAL_HASH_CTX_STS_COMPLETE;
241 }
242
243 void
sha256_single(const void * data,uint32_t digest[])244 sha256_single(const void *data, uint32_t digest[])
245 {
246 uint32_t a, b, c, d, e, f, g, h, t1, t2;
247 uint32_t w[16];
248 uint32_t *ww = (uint32_t *) data;
249
250 a = digest[0];
251 b = digest[1];
252 c = digest[2];
253 d = digest[3];
254 e = digest[4];
255 f = digest[5];
256 g = digest[6];
257 h = digest[7];
258
259 step(0, a, b, c, d, e, f, g, h, 0x428a2f98);
260 step(1, h, a, b, c, d, e, f, g, 0x71374491);
261 step(2, g, h, a, b, c, d, e, f, 0xb5c0fbcf);
262 step(3, f, g, h, a, b, c, d, e, 0xe9b5dba5);
263 step(4, e, f, g, h, a, b, c, d, 0x3956c25b);
264 step(5, d, e, f, g, h, a, b, c, 0x59f111f1);
265 step(6, c, d, e, f, g, h, a, b, 0x923f82a4);
266 step(7, b, c, d, e, f, g, h, a, 0xab1c5ed5);
267 step(8, a, b, c, d, e, f, g, h, 0xd807aa98);
268 step(9, h, a, b, c, d, e, f, g, 0x12835b01);
269 step(10, g, h, a, b, c, d, e, f, 0x243185be);
270 step(11, f, g, h, a, b, c, d, e, 0x550c7dc3);
271 step(12, e, f, g, h, a, b, c, d, 0x72be5d74);
272 step(13, d, e, f, g, h, a, b, c, 0x80deb1fe);
273 step(14, c, d, e, f, g, h, a, b, 0x9bdc06a7);
274 step(15, b, c, d, e, f, g, h, a, 0xc19bf174);
275 step(16, a, b, c, d, e, f, g, h, 0xe49b69c1);
276 step(17, h, a, b, c, d, e, f, g, 0xefbe4786);
277 step(18, g, h, a, b, c, d, e, f, 0x0fc19dc6);
278 step(19, f, g, h, a, b, c, d, e, 0x240ca1cc);
279 step(20, e, f, g, h, a, b, c, d, 0x2de92c6f);
280 step(21, d, e, f, g, h, a, b, c, 0x4a7484aa);
281 step(22, c, d, e, f, g, h, a, b, 0x5cb0a9dc);
282 step(23, b, c, d, e, f, g, h, a, 0x76f988da);
283 step(24, a, b, c, d, e, f, g, h, 0x983e5152);
284 step(25, h, a, b, c, d, e, f, g, 0xa831c66d);
285 step(26, g, h, a, b, c, d, e, f, 0xb00327c8);
286 step(27, f, g, h, a, b, c, d, e, 0xbf597fc7);
287 step(28, e, f, g, h, a, b, c, d, 0xc6e00bf3);
288 step(29, d, e, f, g, h, a, b, c, 0xd5a79147);
289 step(30, c, d, e, f, g, h, a, b, 0x06ca6351);
290 step(31, b, c, d, e, f, g, h, a, 0x14292967);
291 step(32, a, b, c, d, e, f, g, h, 0x27b70a85);
292 step(33, h, a, b, c, d, e, f, g, 0x2e1b2138);
293 step(34, g, h, a, b, c, d, e, f, 0x4d2c6dfc);
294 step(35, f, g, h, a, b, c, d, e, 0x53380d13);
295 step(36, e, f, g, h, a, b, c, d, 0x650a7354);
296 step(37, d, e, f, g, h, a, b, c, 0x766a0abb);
297 step(38, c, d, e, f, g, h, a, b, 0x81c2c92e);
298 step(39, b, c, d, e, f, g, h, a, 0x92722c85);
299 step(40, a, b, c, d, e, f, g, h, 0xa2bfe8a1);
300 step(41, h, a, b, c, d, e, f, g, 0xa81a664b);
301 step(42, g, h, a, b, c, d, e, f, 0xc24b8b70);
302 step(43, f, g, h, a, b, c, d, e, 0xc76c51a3);
303 step(44, e, f, g, h, a, b, c, d, 0xd192e819);
304 step(45, d, e, f, g, h, a, b, c, 0xd6990624);
305 step(46, c, d, e, f, g, h, a, b, 0xf40e3585);
306 step(47, b, c, d, e, f, g, h, a, 0x106aa070);
307 step(48, a, b, c, d, e, f, g, h, 0x19a4c116);
308 step(49, h, a, b, c, d, e, f, g, 0x1e376c08);
309 step(50, g, h, a, b, c, d, e, f, 0x2748774c);
310 step(51, f, g, h, a, b, c, d, e, 0x34b0bcb5);
311 step(52, e, f, g, h, a, b, c, d, 0x391c0cb3);
312 step(53, d, e, f, g, h, a, b, c, 0x4ed8aa4a);
313 step(54, c, d, e, f, g, h, a, b, 0x5b9cca4f);
314 step(55, b, c, d, e, f, g, h, a, 0x682e6ff3);
315 step(56, a, b, c, d, e, f, g, h, 0x748f82ee);
316 step(57, h, a, b, c, d, e, f, g, 0x78a5636f);
317 step(58, g, h, a, b, c, d, e, f, 0x84c87814);
318 step(59, f, g, h, a, b, c, d, e, 0x8cc70208);
319 step(60, e, f, g, h, a, b, c, d, 0x90befffa);
320 step(61, d, e, f, g, h, a, b, c, 0xa4506ceb);
321 step(62, c, d, e, f, g, h, a, b, 0xbef9a3f7);
322 step(63, b, c, d, e, f, g, h, a, 0xc67178f2);
323
324 digest[0] += a;
325 digest[1] += b;
326 digest[2] += c;
327 digest[3] += d;
328 digest[4] += e;
329 digest[5] += f;
330 digest[6] += g;
331 digest[7] += h;
332 }
333
334 static inline void
hash_init_digest(ISAL_SHA256_WORD_T * digest)335 hash_init_digest(ISAL_SHA256_WORD_T *digest)
336 {
337 static const ISAL_SHA256_WORD_T hash_initial_digest[ISAL_SHA256_DIGEST_NWORDS] = {
338 ISAL_SHA256_INITIAL_DIGEST
339 };
340 memcpy_fixedlen(digest, hash_initial_digest, sizeof(hash_initial_digest));
341 }
342
343 struct slver {
344 uint16_t snum;
345 uint8_t ver;
346 uint8_t core;
347 };
348 struct slver _sha256_ctx_mgr_init_base_slver_000002f0;
349 struct slver _sha256_ctx_mgr_init_base_slver = { 0x02f0, 0x00, 0x00 };
350
351 struct slver _sha256_ctx_mgr_submit_base_slver_000002f1;
352 struct slver _sha256_ctx_mgr_submit_base_slver = { 0x02f1, 0x00, 0x00 };
353
354 struct slver _sha256_ctx_mgr_flush_base_slver_000002f2;
355 struct slver _sha256_ctx_mgr_flush_base_slver = { 0x02f2, 0x00, 0x00 };
356