1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #include "sha512_mb_internal.h"
31 #include "memcpy_inline.h"
32 #include "endian_helper.h"
33
34 #ifdef _MSC_VER
35 #include <intrin.h>
36 #define inline __inline
37 #endif
38
39 static inline void
40 hash_init_digest(ISAL_SHA512_WORD_T *digest);
41 static inline uint32_t
42 hash_pad(uint8_t padblock[ISAL_SHA512_BLOCK_SIZE * 2], uint64_t total_len);
43 static ISAL_SHA512_HASH_CTX *
44 sha512_ctx_mgr_resubmit(ISAL_SHA512_HASH_CTX_MGR *mgr, ISAL_SHA512_HASH_CTX *ctx);
45
46 void
_sha512_ctx_mgr_init_sse(ISAL_SHA512_HASH_CTX_MGR * mgr)47 _sha512_ctx_mgr_init_sse(ISAL_SHA512_HASH_CTX_MGR *mgr)
48 {
49 _sha512_mb_mgr_init_sse(&mgr->mgr);
50 }
51
52 ISAL_SHA512_HASH_CTX *
_sha512_ctx_mgr_submit_sse(ISAL_SHA512_HASH_CTX_MGR * mgr,ISAL_SHA512_HASH_CTX * ctx,const void * buffer,uint32_t len,ISAL_HASH_CTX_FLAG flags)53 _sha512_ctx_mgr_submit_sse(ISAL_SHA512_HASH_CTX_MGR *mgr, ISAL_SHA512_HASH_CTX *ctx,
54 const void *buffer, uint32_t len, ISAL_HASH_CTX_FLAG flags)
55 {
56 if (flags & (~ISAL_HASH_ENTIRE)) {
57 // User should not pass anything other than FIRST, UPDATE, or LAST
58 ctx->error = ISAL_HASH_CTX_ERROR_INVALID_FLAGS;
59 return ctx;
60 }
61
62 if (ctx->status & ISAL_HASH_CTX_STS_PROCESSING) {
63 // Cannot submit to a currently processing job.
64 ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_PROCESSING;
65 return ctx;
66 }
67
68 if ((ctx->status & ISAL_HASH_CTX_STS_COMPLETE) && !(flags & ISAL_HASH_FIRST)) {
69 // Cannot update a finished job.
70 ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_COMPLETED;
71 return ctx;
72 }
73
74 if (flags & ISAL_HASH_FIRST) {
75 // Init digest
76 hash_init_digest(ctx->job.result_digest);
77
78 // Reset byte counter
79 ctx->total_length = 0;
80
81 // Clear extra blocks
82 ctx->partial_block_buffer_length = 0;
83 }
84 // If we made it here, there were no errors during this call to submit
85 ctx->error = ISAL_HASH_CTX_ERROR_NONE;
86
87 // Store buffer ptr info from user
88 ctx->incoming_buffer = buffer;
89 ctx->incoming_buffer_length = len;
90
91 // Store the user's request flags and mark this ctx as currently being processed.
92 ctx->status = (flags & ISAL_HASH_LAST) ? (ISAL_HASH_CTX_STS) (ISAL_HASH_CTX_STS_PROCESSING |
93 ISAL_HASH_CTX_STS_LAST)
94 : ISAL_HASH_CTX_STS_PROCESSING;
95
96 // Advance byte counter
97 ctx->total_length += len;
98
99 // If there is anything currently buffered in the extra blocks, append to it until it
100 // contains a whole block. Or if the user's buffer contains less than a whole block, append
101 // as much as possible to the extra block.
102 if ((ctx->partial_block_buffer_length) | (len < ISAL_SHA512_BLOCK_SIZE)) {
103 // Compute how many bytes to copy from user buffer into extra block
104 uint32_t copy_len = ISAL_SHA512_BLOCK_SIZE - ctx->partial_block_buffer_length;
105 if (len < copy_len)
106 copy_len = len;
107
108 if (copy_len) {
109 // Copy and update relevant pointers and counters
110 memcpy_varlen(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
111 buffer, copy_len);
112
113 ctx->partial_block_buffer_length += copy_len;
114 ctx->incoming_buffer = (const void *) ((const char *) buffer + copy_len);
115 ctx->incoming_buffer_length = len - copy_len;
116 }
117 // The extra block should never contain more than 1 block here
118 assert(ctx->partial_block_buffer_length <= ISAL_SHA512_BLOCK_SIZE);
119
120 // If the extra block buffer contains exactly 1 block, it can be hashed.
121 if (ctx->partial_block_buffer_length >= ISAL_SHA512_BLOCK_SIZE) {
122 ctx->partial_block_buffer_length = 0;
123
124 ctx->job.buffer = ctx->partial_block_buffer;
125 ctx->job.len = 1;
126
127 ctx = (ISAL_SHA512_HASH_CTX *) _sha512_mb_mgr_submit_sse(&mgr->mgr,
128 &ctx->job);
129 }
130 }
131
132 return sha512_ctx_mgr_resubmit(mgr, ctx);
133 }
134
135 ISAL_SHA512_HASH_CTX *
_sha512_ctx_mgr_flush_sse(ISAL_SHA512_HASH_CTX_MGR * mgr)136 _sha512_ctx_mgr_flush_sse(ISAL_SHA512_HASH_CTX_MGR *mgr)
137 {
138 ISAL_SHA512_HASH_CTX *ctx;
139
140 while (1) {
141 ctx = (ISAL_SHA512_HASH_CTX *) _sha512_mb_mgr_flush_sse(&mgr->mgr);
142
143 // If flush returned 0, there are no more jobs in flight.
144 if (!ctx)
145 return NULL;
146
147 // If flush returned a job, verify that it is safe to return to the user.
148 // If it is not ready, resubmit the job to finish processing.
149 ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
150
151 // If sha512_ctx_mgr_resubmit returned a job, it is ready to be returned.
152 if (ctx)
153 return ctx;
154
155 // Otherwise, all jobs currently being managed by the ISAL_SHA512_HASH_CTX_MGR still
156 // need processing. Loop.
157 }
158 }
159
160 static ISAL_SHA512_HASH_CTX *
sha512_ctx_mgr_resubmit(ISAL_SHA512_HASH_CTX_MGR * mgr,ISAL_SHA512_HASH_CTX * ctx)161 sha512_ctx_mgr_resubmit(ISAL_SHA512_HASH_CTX_MGR *mgr, ISAL_SHA512_HASH_CTX *ctx)
162 {
163 while (ctx) {
164 if (ctx->status & ISAL_HASH_CTX_STS_COMPLETE) {
165 ctx->status = ISAL_HASH_CTX_STS_COMPLETE; // Clear PROCESSING bit
166 return ctx;
167 }
168 // If the extra blocks are empty, begin hashing what remains in the user's buffer.
169 if (ctx->partial_block_buffer_length == 0 && ctx->incoming_buffer_length) {
170 const void *buffer = ctx->incoming_buffer;
171 uint32_t len = ctx->incoming_buffer_length;
172
173 // Only entire blocks can be hashed. Copy remainder to extra blocks buffer.
174 uint32_t copy_len = len & (ISAL_SHA512_BLOCK_SIZE - 1);
175
176 if (copy_len) {
177 len -= copy_len;
178 memcpy_varlen(ctx->partial_block_buffer,
179 ((const char *) buffer + len), copy_len);
180 ctx->partial_block_buffer_length = copy_len;
181 }
182
183 ctx->incoming_buffer_length = 0;
184
185 // len should be a multiple of the block size now
186 assert((len % ISAL_SHA512_BLOCK_SIZE) == 0);
187
188 // Set len to the number of blocks to be hashed in the user's buffer
189 len >>= ISAL_SHA512_LOG2_BLOCK_SIZE;
190
191 if (len) {
192 ctx->job.buffer = (uint8_t *) buffer;
193 ctx->job.len = len;
194 ctx = (ISAL_SHA512_HASH_CTX *) _sha512_mb_mgr_submit_sse(&mgr->mgr,
195 &ctx->job);
196 continue;
197 }
198 }
199 // If the extra blocks are not empty, then we are either on the last block(s)
200 // or we need more user input before continuing.
201 if (ctx->status & ISAL_HASH_CTX_STS_LAST) {
202 uint8_t *buf = ctx->partial_block_buffer;
203 uint32_t n_extra_blocks = hash_pad(buf, ctx->total_length);
204
205 ctx->status = (ISAL_HASH_CTX_STS) (ISAL_HASH_CTX_STS_PROCESSING |
206 ISAL_HASH_CTX_STS_COMPLETE);
207 ctx->job.buffer = buf;
208 ctx->job.len = (uint32_t) n_extra_blocks;
209 ctx = (ISAL_SHA512_HASH_CTX *) _sha512_mb_mgr_submit_sse(&mgr->mgr,
210 &ctx->job);
211 continue;
212 }
213
214 if (ctx)
215 ctx->status = ISAL_HASH_CTX_STS_IDLE;
216 return ctx;
217 }
218
219 return NULL;
220 }
221
222 static inline void
hash_init_digest(ISAL_SHA512_WORD_T * digest)223 hash_init_digest(ISAL_SHA512_WORD_T *digest)
224 {
225 static const ISAL_SHA512_WORD_T hash_initial_digest[ISAL_SHA512_DIGEST_NWORDS] = {
226 ISAL_SHA512_INITIAL_DIGEST
227 };
228 memcpy_fixedlen(digest, hash_initial_digest, sizeof(hash_initial_digest));
229 }
230
231 static inline uint32_t
hash_pad(uint8_t padblock[ISAL_SHA512_BLOCK_SIZE * 2],uint64_t total_len)232 hash_pad(uint8_t padblock[ISAL_SHA512_BLOCK_SIZE * 2], uint64_t total_len)
233 {
234 uint32_t i = (uint32_t) (total_len & (ISAL_SHA512_BLOCK_SIZE - 1));
235
236 memclr_fixedlen(&padblock[i], ISAL_SHA512_BLOCK_SIZE);
237 padblock[i] = 0x80;
238
239 // Move i to the end of either 1st or 2nd extra block depending on length
240 i += ((ISAL_SHA512_BLOCK_SIZE - 1) &
241 (0 - (total_len + ISAL_SHA512_PADLENGTHFIELD_SIZE + 1))) +
242 1 + ISAL_SHA512_PADLENGTHFIELD_SIZE;
243
244 #if ISAL_SHA512_PADLENGTHFIELD_SIZE == 16
245 *((uint64_t *) &padblock[i - 16]) = 0;
246 #endif
247
248 *((uint64_t *) &padblock[i - 8]) = to_be64((uint64_t) total_len << 3);
249
250 return i >> ISAL_SHA512_LOG2_BLOCK_SIZE; // Number of extra blocks to hash
251 }
252
253 struct slver {
254 uint16_t snum;
255 uint8_t ver;
256 uint8_t core;
257 };
258 struct slver _sha512_ctx_mgr_init_sse_slver_00020163;
259 struct slver _sha512_ctx_mgr_init_sse_slver = { 0x0163, 0x02, 0x00 };
260
261 struct slver _sha512_ctx_mgr_submit_sse_slver_00020164;
262 struct slver _sha512_ctx_mgr_submit_sse_slver = { 0x0164, 0x02, 0x00 };
263
264 struct slver _sha512_ctx_mgr_flush_sse_slver_00020165;
265 struct slver _sha512_ctx_mgr_flush_sse_slver = { 0x0165, 0x02, 0x00 };
266