1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #include "sha1_mb_internal.h"
31 #include "memcpy_inline.h"
32 #include "endian_helper.h"
33
34 #ifdef _MSC_VER
35 #include <intrin.h>
36 #define inline __inline
37 #endif
38
39 static inline void
40 hash_init_digest(ISAL_SHA1_WORD_T *digest);
41 static inline uint32_t
42 hash_pad(uint8_t padblock[ISAL_SHA1_BLOCK_SIZE * 2], uint64_t total_len);
43 static ISAL_SHA1_HASH_CTX *
44 sha1_ctx_mgr_resubmit(ISAL_SHA1_HASH_CTX_MGR *mgr, ISAL_SHA1_HASH_CTX *ctx);
45
46 void
_sha1_ctx_mgr_init_sse(ISAL_SHA1_HASH_CTX_MGR * mgr)47 _sha1_ctx_mgr_init_sse(ISAL_SHA1_HASH_CTX_MGR *mgr)
48 {
49 _sha1_mb_mgr_init_sse(&mgr->mgr);
50 }
51
52 ISAL_SHA1_HASH_CTX *
_sha1_ctx_mgr_submit_sse(ISAL_SHA1_HASH_CTX_MGR * mgr,ISAL_SHA1_HASH_CTX * ctx,const void * buffer,uint32_t len,ISAL_HASH_CTX_FLAG flags)53 _sha1_ctx_mgr_submit_sse(ISAL_SHA1_HASH_CTX_MGR *mgr, ISAL_SHA1_HASH_CTX *ctx, const void *buffer,
54 uint32_t len, ISAL_HASH_CTX_FLAG flags)
55 {
56 if (flags & (~ISAL_HASH_ENTIRE)) {
57 // User should not pass anything other than FIRST, UPDATE, or LAST
58 ctx->error = ISAL_HASH_CTX_ERROR_INVALID_FLAGS;
59 return ctx;
60 }
61
62 if (ctx->status & ISAL_HASH_CTX_STS_PROCESSING) {
63 // Cannot submit to a currently processing job.
64 ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_PROCESSING;
65 return ctx;
66 }
67
68 if ((ctx->status & ISAL_HASH_CTX_STS_COMPLETE) && !(flags & ISAL_HASH_FIRST)) {
69 // Cannot update a finished job.
70 ctx->error = ISAL_HASH_CTX_ERROR_ALREADY_COMPLETED;
71 return ctx;
72 }
73
74 if (flags & ISAL_HASH_FIRST) {
75 // Init digest
76 hash_init_digest(ctx->job.result_digest);
77
78 // Reset byte counter
79 ctx->total_length = 0;
80
81 // Clear extra blocks
82 ctx->partial_block_buffer_length = 0;
83 }
84 // If we made it here, there were no errors during this call to submit
85 ctx->error = ISAL_HASH_CTX_ERROR_NONE;
86
87 // Store buffer ptr info from user
88 ctx->incoming_buffer = buffer;
89 ctx->incoming_buffer_length = len;
90
91 // Store the user's request flags and mark this ctx as currently being processed.
92 ctx->status = (flags & ISAL_HASH_LAST) ? (ISAL_HASH_CTX_STS) (ISAL_HASH_CTX_STS_PROCESSING |
93 ISAL_HASH_CTX_STS_LAST)
94 : ISAL_HASH_CTX_STS_PROCESSING;
95
96 // Advance byte counter
97 ctx->total_length += len;
98
99 // If there is anything currently buffered in the extra blocks, append to it until it
100 // contains a whole block. Or if the user's buffer contains less than a whole block, append
101 // as much as possible to the extra block.
102 if ((ctx->partial_block_buffer_length) | (len < ISAL_SHA1_BLOCK_SIZE)) {
103 // Compute how many bytes to copy from user buffer into extra block
104 uint32_t copy_len = ISAL_SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length;
105 if (len < copy_len)
106 copy_len = len;
107
108 if (copy_len) {
109 // Copy and update relevant pointers and counters
110 memcpy_varlen(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
111 buffer, copy_len);
112
113 ctx->partial_block_buffer_length += copy_len;
114 ctx->incoming_buffer = (const void *) ((const char *) buffer + copy_len);
115 ctx->incoming_buffer_length = len - copy_len;
116 }
117 // The extra block should never contain more than 1 block here
118 assert(ctx->partial_block_buffer_length <= ISAL_SHA1_BLOCK_SIZE);
119
120 // If the extra block buffer contains exactly 1 block, it can be hashed.
121 if (ctx->partial_block_buffer_length >= ISAL_SHA1_BLOCK_SIZE) {
122 ctx->partial_block_buffer_length = 0;
123
124 ctx->job.buffer = ctx->partial_block_buffer;
125 ctx->job.len = 1;
126
127 ctx = (ISAL_SHA1_HASH_CTX *) _sha1_mb_mgr_submit_sse(&mgr->mgr, &ctx->job);
128 }
129 }
130
131 return sha1_ctx_mgr_resubmit(mgr, ctx);
132 }
133
134 ISAL_SHA1_HASH_CTX *
_sha1_ctx_mgr_flush_sse(ISAL_SHA1_HASH_CTX_MGR * mgr)135 _sha1_ctx_mgr_flush_sse(ISAL_SHA1_HASH_CTX_MGR *mgr)
136 {
137 ISAL_SHA1_HASH_CTX *ctx;
138
139 while (1) {
140 ctx = (ISAL_SHA1_HASH_CTX *) _sha1_mb_mgr_flush_sse(&mgr->mgr);
141
142 // If flush returned 0, there are no more jobs in flight.
143 if (!ctx)
144 return NULL;
145
146 // If flush returned a job, verify that it is safe to return to the user.
147 // If it is not ready, resubmit the job to finish processing.
148 ctx = sha1_ctx_mgr_resubmit(mgr, ctx);
149
150 // If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
151 if (ctx)
152 return ctx;
153
154 // Otherwise, all jobs currently being managed by the ISAL_SHA1_HASH_CTX_MGR still
155 // need processing. Loop.
156 }
157 }
158
159 static ISAL_SHA1_HASH_CTX *
sha1_ctx_mgr_resubmit(ISAL_SHA1_HASH_CTX_MGR * mgr,ISAL_SHA1_HASH_CTX * ctx)160 sha1_ctx_mgr_resubmit(ISAL_SHA1_HASH_CTX_MGR *mgr, ISAL_SHA1_HASH_CTX *ctx)
161 {
162 while (ctx) {
163
164 if (ctx->status & ISAL_HASH_CTX_STS_COMPLETE) {
165 ctx->status = ISAL_HASH_CTX_STS_COMPLETE; // Clear PROCESSING bit
166 return ctx;
167 }
168 // If the extra blocks are empty, begin hashing what remains in the user's buffer.
169 if (ctx->partial_block_buffer_length == 0 && ctx->incoming_buffer_length) {
170 const void *buffer = ctx->incoming_buffer;
171 uint32_t len = ctx->incoming_buffer_length;
172
173 // Only entire blocks can be hashed. Copy remainder to extra blocks buffer.
174 uint32_t copy_len = len & (ISAL_SHA1_BLOCK_SIZE - 1);
175
176 if (copy_len) {
177 len -= copy_len;
178 memcpy_varlen(ctx->partial_block_buffer,
179 ((const char *) buffer + len), copy_len);
180 ctx->partial_block_buffer_length = copy_len;
181 }
182
183 ctx->incoming_buffer_length = 0;
184
185 // len should be a multiple of the block size now
186 assert((len % ISAL_SHA1_BLOCK_SIZE) == 0);
187
188 // Set len to the number of blocks to be hashed in the user's buffer
189 len >>= ISAL_SHA1_LOG2_BLOCK_SIZE;
190
191 if (len) {
192 ctx->job.buffer = (uint8_t *) buffer;
193 ctx->job.len = len;
194 ctx = (ISAL_SHA1_HASH_CTX *) _sha1_mb_mgr_submit_sse(&mgr->mgr,
195 &ctx->job);
196 continue;
197 }
198 }
199 // If the extra blocks are not empty, then we are either on the last block(s)
200 // or we need more user input before continuing.
201 if (ctx->status & ISAL_HASH_CTX_STS_LAST) {
202 uint8_t *buf = ctx->partial_block_buffer;
203 uint32_t n_extra_blocks = hash_pad(buf, ctx->total_length);
204
205 ctx->status = (ISAL_HASH_CTX_STS) (ISAL_HASH_CTX_STS_PROCESSING |
206 ISAL_HASH_CTX_STS_COMPLETE);
207 ctx->job.buffer = buf;
208 ctx->job.len = (uint32_t) n_extra_blocks;
209 ctx = (ISAL_SHA1_HASH_CTX *) _sha1_mb_mgr_submit_sse(&mgr->mgr, &ctx->job);
210 continue;
211 }
212
213 if (ctx)
214 ctx->status = ISAL_HASH_CTX_STS_IDLE;
215 return ctx;
216 }
217
218 return NULL;
219 }
220
221 static inline void
hash_init_digest(ISAL_SHA1_WORD_T * digest)222 hash_init_digest(ISAL_SHA1_WORD_T *digest)
223 {
224 static const ISAL_SHA1_WORD_T hash_initial_digest[ISAL_SHA1_DIGEST_NWORDS] = {
225 ISAL_SHA1_INITIAL_DIGEST
226 };
227 memcpy_fixedlen(digest, hash_initial_digest, sizeof(hash_initial_digest));
228 }
229
230 static inline uint32_t
hash_pad(uint8_t padblock[ISAL_SHA1_BLOCK_SIZE * 2],uint64_t total_len)231 hash_pad(uint8_t padblock[ISAL_SHA1_BLOCK_SIZE * 2], uint64_t total_len)
232 {
233 uint32_t i = (uint32_t) (total_len & (ISAL_SHA1_BLOCK_SIZE - 1));
234
235 memclr_fixedlen(&padblock[i], ISAL_SHA1_BLOCK_SIZE);
236 padblock[i] = 0x80;
237
238 // Move i to the end of either 1st or 2nd extra block depending on length
239 i += ((ISAL_SHA1_BLOCK_SIZE - 1) & (0 - (total_len + ISAL_SHA1_PADLENGTHFIELD_SIZE + 1))) +
240 1 + ISAL_SHA1_PADLENGTHFIELD_SIZE;
241
242 #if ISAL_SHA1_PADLENGTHFIELD_SIZE == 16
243 *((uint64_t *) &padblock[i - 16]) = 0;
244 #endif
245
246 *((uint64_t *) &padblock[i - 8]) = to_be64((uint64_t) total_len << 3);
247
248 return i >> ISAL_SHA1_LOG2_BLOCK_SIZE; // Number of extra blocks to hash
249 }
250
251 struct slver {
252 uint16_t snum;
253 uint8_t ver;
254 uint8_t core;
255 };
256 struct slver _sha1_ctx_mgr_init_sse_slver_00020139;
257 struct slver _sha1_ctx_mgr_init_sse_slver = { 0x0139, 0x02, 0x00 };
258
259 struct slver _sha1_ctx_mgr_submit_sse_slver_00020140;
260 struct slver _sha1_ctx_mgr_submit_sse_slver = { 0x0140, 0x02, 0x00 };
261
262 struct slver _sha1_ctx_mgr_flush_sse_slver_00020141;
263 struct slver _sha1_ctx_mgr_flush_sse_slver = { 0x0141, 0x02, 0x00 };
264