1 /*
2 * Copyright (c) 2016 Thomas Pornin <pornin@bolet.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files (the
6 * "Software"), to deal in the Software without restriction, including
7 * without limitation the rights to use, copy, modify, merge, publish,
8 * distribute, sublicense, and/or sell copies of the Software, and to
9 * permit persons to whom the Software is furnished to do so, subject to
10 * the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
19 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
20 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(1, "$NetBSD: aes_sse2.c,v 1.2 2020/06/30 20:32:11 riastradh Exp $");
27
28 #include <sys/types.h>
29
30 #ifdef _KERNEL
31 #include <lib/libkern/libkern.h>
32 #else
33 #include <stdint.h>
34 #include <string.h>
35 #endif
36
37 #include "aes_sse2_impl.h"
38
39 static void
br_range_dec32le(uint32_t * p32,size_t nwords,const void * v)40 br_range_dec32le(uint32_t *p32, size_t nwords, const void *v)
41 {
42 const uint8_t *p8 = v;
43
44 while (nwords --> 0) {
45 uint32_t x0 = *p8++;
46 uint32_t x1 = *p8++;
47 uint32_t x2 = *p8++;
48 uint32_t x3 = *p8++;
49
50 *p32++ = x0 | (x1 << 8) | (x2 << 16) | (x3 << 24);
51 }
52 }
53
54 void
aes_sse2_bitslice_Sbox(__m128i q[static4])55 aes_sse2_bitslice_Sbox(__m128i q[static 4])
56 {
57 __m128i x0, x1, x2, x3, x4, x5, x6, x7;
58 __m128i y1, y2, y3, y4, y5, y6, y7, y8, y9;
59 __m128i y10, y11, y12, y13, y14, y15, y16, y17, y18, y19;
60 __m128i y20, y21;
61 __m128i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9;
62 __m128i z10, z11, z12, z13, z14, z15, z16, z17;
63 __m128i t0, t1, t2, t3, t4, t5, t6, t7, t8, t9;
64 __m128i t10, t11, t12, t13, t14, t15, t16, t17, t18, t19;
65 __m128i t20, t21, t22, t23, t24, t25, t26, t27, t28, t29;
66 __m128i t30, t31, t32, t33, t34, t35, t36, t37, t38, t39;
67 __m128i t40, t41, t42, t43, t44, t45, t46, t47, t48, t49;
68 __m128i t50, t51, t52, t53, t54, t55, t56, t57, t58, t59;
69 __m128i t60, t61, t62, t63, t64, t65, t66, t67;
70 __m128i s0, s1, s2, s3, s4, s5, s6, s7;
71
72 x0 = _mm_shuffle_epi32(q[3], 0x0e);
73 x1 = _mm_shuffle_epi32(q[2], 0x0e);
74 x2 = _mm_shuffle_epi32(q[1], 0x0e);
75 x3 = _mm_shuffle_epi32(q[0], 0x0e);
76 x4 = q[3];
77 x5 = q[2];
78 x6 = q[1];
79 x7 = q[0];
80
81 /*
82 * Top linear transformation.
83 */
84 y14 = x3 ^ x5;
85 y13 = x0 ^ x6;
86 y9 = x0 ^ x3;
87 y8 = x0 ^ x5;
88 t0 = x1 ^ x2;
89 y1 = t0 ^ x7;
90 y4 = y1 ^ x3;
91 y12 = y13 ^ y14;
92 y2 = y1 ^ x0;
93 y5 = y1 ^ x6;
94 y3 = y5 ^ y8;
95 t1 = x4 ^ y12;
96 y15 = t1 ^ x5;
97 y20 = t1 ^ x1;
98 y6 = y15 ^ x7;
99 y10 = y15 ^ t0;
100 y11 = y20 ^ y9;
101 y7 = x7 ^ y11;
102 y17 = y10 ^ y11;
103 y19 = y10 ^ y8;
104 y16 = t0 ^ y11;
105 y21 = y13 ^ y16;
106 y18 = x0 ^ y16;
107
108 /*
109 * Non-linear section.
110 */
111 t2 = y12 & y15;
112 t3 = y3 & y6;
113 t4 = t3 ^ t2;
114 t5 = y4 & x7;
115 t6 = t5 ^ t2;
116 t7 = y13 & y16;
117 t8 = y5 & y1;
118 t9 = t8 ^ t7;
119 t10 = y2 & y7;
120 t11 = t10 ^ t7;
121 t12 = y9 & y11;
122 t13 = y14 & y17;
123 t14 = t13 ^ t12;
124 t15 = y8 & y10;
125 t16 = t15 ^ t12;
126 t17 = t4 ^ t14;
127 t18 = t6 ^ t16;
128 t19 = t9 ^ t14;
129 t20 = t11 ^ t16;
130 t21 = t17 ^ y20;
131 t22 = t18 ^ y19;
132 t23 = t19 ^ y21;
133 t24 = t20 ^ y18;
134
135 t25 = t21 ^ t22;
136 t26 = t21 & t23;
137 t27 = t24 ^ t26;
138 t28 = t25 & t27;
139 t29 = t28 ^ t22;
140 t30 = t23 ^ t24;
141 t31 = t22 ^ t26;
142 t32 = t31 & t30;
143 t33 = t32 ^ t24;
144 t34 = t23 ^ t33;
145 t35 = t27 ^ t33;
146 t36 = t24 & t35;
147 t37 = t36 ^ t34;
148 t38 = t27 ^ t36;
149 t39 = t29 & t38;
150 t40 = t25 ^ t39;
151
152 t41 = t40 ^ t37;
153 t42 = t29 ^ t33;
154 t43 = t29 ^ t40;
155 t44 = t33 ^ t37;
156 t45 = t42 ^ t41;
157 z0 = t44 & y15;
158 z1 = t37 & y6;
159 z2 = t33 & x7;
160 z3 = t43 & y16;
161 z4 = t40 & y1;
162 z5 = t29 & y7;
163 z6 = t42 & y11;
164 z7 = t45 & y17;
165 z8 = t41 & y10;
166 z9 = t44 & y12;
167 z10 = t37 & y3;
168 z11 = t33 & y4;
169 z12 = t43 & y13;
170 z13 = t40 & y5;
171 z14 = t29 & y2;
172 z15 = t42 & y9;
173 z16 = t45 & y14;
174 z17 = t41 & y8;
175
176 /*
177 * Bottom linear transformation.
178 */
179 t46 = z15 ^ z16;
180 t47 = z10 ^ z11;
181 t48 = z5 ^ z13;
182 t49 = z9 ^ z10;
183 t50 = z2 ^ z12;
184 t51 = z2 ^ z5;
185 t52 = z7 ^ z8;
186 t53 = z0 ^ z3;
187 t54 = z6 ^ z7;
188 t55 = z16 ^ z17;
189 t56 = z12 ^ t48;
190 t57 = t50 ^ t53;
191 t58 = z4 ^ t46;
192 t59 = z3 ^ t54;
193 t60 = t46 ^ t57;
194 t61 = z14 ^ t57;
195 t62 = t52 ^ t58;
196 t63 = t49 ^ t58;
197 t64 = z4 ^ t59;
198 t65 = t61 ^ t62;
199 t66 = z1 ^ t63;
200 s0 = t59 ^ t63;
201 s6 = t56 ^ ~t62;
202 s7 = t48 ^ ~t60;
203 t67 = t64 ^ t65;
204 s3 = t53 ^ t66;
205 s4 = t51 ^ t66;
206 s5 = t47 ^ t65;
207 s1 = t64 ^ ~s3;
208 s2 = t55 ^ ~t67;
209
210 q[3] = _mm_unpacklo_epi64(s4, s0);
211 q[2] = _mm_unpacklo_epi64(s5, s1);
212 q[1] = _mm_unpacklo_epi64(s6, s2);
213 q[0] = _mm_unpacklo_epi64(s7, s3);
214 }
215
216 void
aes_sse2_ortho(__m128i q[static4])217 aes_sse2_ortho(__m128i q[static 4])
218 {
219 #define SWAPN(cl, ch, s, x, y) do { \
220 __m128i a, b; \
221 a = (x); \
222 b = (y); \
223 (x) = (a & _mm_set1_epi64x(cl)) | \
224 _mm_slli_epi64(b & _mm_set1_epi64x(cl), (s)); \
225 (y) = _mm_srli_epi64(a & _mm_set1_epi64x(ch), (s)) | \
226 (b & _mm_set1_epi64x(ch)); \
227 } while (0)
228
229 #define SWAP2(x, y) SWAPN(0x5555555555555555, 0xAAAAAAAAAAAAAAAA, 1, x, y)
230 #define SWAP4(x, y) SWAPN(0x3333333333333333, 0xCCCCCCCCCCCCCCCC, 2, x, y)
231 #define SWAP8(x, y) SWAPN(0x0F0F0F0F0F0F0F0F, 0xF0F0F0F0F0F0F0F0, 4, x, y)
232
233 SWAP2(q[0], q[1]);
234 SWAP2(q[2], q[3]);
235
236 SWAP4(q[0], q[2]);
237 SWAP4(q[1], q[3]);
238
239 __m128i q0 = q[0];
240 __m128i q1 = q[1];
241 __m128i q2 = q[2];
242 __m128i q3 = q[3];
243 __m128i q4 = _mm_shuffle_epi32(q[0], 0x0e);
244 __m128i q5 = _mm_shuffle_epi32(q[1], 0x0e);
245 __m128i q6 = _mm_shuffle_epi32(q[2], 0x0e);
246 __m128i q7 = _mm_shuffle_epi32(q[3], 0x0e);
247 SWAP8(q0, q4);
248 SWAP8(q1, q5);
249 SWAP8(q2, q6);
250 SWAP8(q3, q7);
251 q[0] = _mm_unpacklo_epi64(q0, q4);
252 q[1] = _mm_unpacklo_epi64(q1, q5);
253 q[2] = _mm_unpacklo_epi64(q2, q6);
254 q[3] = _mm_unpacklo_epi64(q3, q7);
255 }
256
257 __m128i
aes_sse2_interleave_in(__m128i w)258 aes_sse2_interleave_in(__m128i w)
259 {
260 __m128i lo, hi;
261
262 lo = _mm_shuffle_epi32(w, 0x10);
263 hi = _mm_shuffle_epi32(w, 0x32);
264 lo &= _mm_set1_epi64x(0x00000000FFFFFFFF);
265 hi &= _mm_set1_epi64x(0x00000000FFFFFFFF);
266 lo |= _mm_slli_epi64(lo, 16);
267 hi |= _mm_slli_epi64(hi, 16);
268 lo &= _mm_set1_epi32(0x0000FFFF);
269 hi &= _mm_set1_epi32(0x0000FFFF);
270 lo |= _mm_slli_epi64(lo, 8);
271 hi |= _mm_slli_epi64(hi, 8);
272 lo &= _mm_set1_epi16(0x00FF);
273 hi &= _mm_set1_epi16(0x00FF);
274 return lo | _mm_slli_epi64(hi, 8);
275 }
276
277 __m128i
aes_sse2_interleave_out(__m128i q)278 aes_sse2_interleave_out(__m128i q)
279 {
280 __m128i lo, hi;
281
282 lo = q;
283 hi = _mm_srli_si128(q, 1);
284 lo &= _mm_set1_epi16(0x00FF);
285 hi &= _mm_set1_epi16(0x00FF);
286 lo |= _mm_srli_epi64(lo, 8);
287 hi |= _mm_srli_epi64(hi, 8);
288 lo &= _mm_set1_epi32(0x0000FFFF);
289 hi &= _mm_set1_epi32(0x0000FFFF);
290 lo |= _mm_srli_epi64(lo, 16);
291 hi |= _mm_srli_epi64(hi, 16);
292 return (__m128i)_mm_shuffle_ps((__m128)lo, (__m128)hi, 0x88);
293 }
294
295 static const unsigned char Rcon[] = {
296 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
297 };
298
299 static uint32_t
sub_word(uint32_t x)300 sub_word(uint32_t x)
301 {
302 __m128i q[4];
303 uint32_t y;
304
305 memset(q, 0, sizeof(q));
306 q[0] = _mm_loadu_si32(&x);
307 aes_sse2_ortho(q);
308 aes_sse2_bitslice_Sbox(q);
309 aes_sse2_ortho(q);
310 _mm_storeu_si32(&y, q[0]);
311 return y;
312 }
313
314 unsigned
aes_sse2_keysched(uint64_t * comp_skey,const void * key,size_t key_len)315 aes_sse2_keysched(uint64_t *comp_skey, const void *key, size_t key_len)
316 {
317 unsigned num_rounds;
318 int i, j, k, nk, nkf;
319 uint32_t tmp;
320 uint32_t skey[60];
321
322 switch (key_len) {
323 case 16:
324 num_rounds = 10;
325 break;
326 case 24:
327 num_rounds = 12;
328 break;
329 case 32:
330 num_rounds = 14;
331 break;
332 default:
333 /* abort(); */
334 return 0;
335 }
336 nk = (int)(key_len >> 2);
337 nkf = (int)((num_rounds + 1) << 2);
338 br_range_dec32le(skey, (key_len >> 2), key);
339 tmp = skey[(key_len >> 2) - 1];
340 for (i = nk, j = 0, k = 0; i < nkf; i ++) {
341 if (j == 0) {
342 tmp = (tmp << 24) | (tmp >> 8);
343 tmp = sub_word(tmp) ^ Rcon[k];
344 } else if (nk > 6 && j == 4) {
345 tmp = sub_word(tmp);
346 }
347 tmp ^= skey[i - nk];
348 skey[i] = tmp;
349 if (++ j == nk) {
350 j = 0;
351 k ++;
352 }
353 }
354
355 for (i = 0, j = 0; i < nkf; i += 4, j += 2) {
356 __m128i q[4], q0, q1, q2, q3, q4, q5, q6, q7;
357 __m128i w;
358
359 w = _mm_loadu_epi8(skey + i);
360 q[0] = q[1] = q[2] = q[3] = aes_sse2_interleave_in(w);
361 aes_sse2_ortho(q);
362 q0 = q[0] & _mm_set1_epi64x(0x1111111111111111);
363 q1 = q[1] & _mm_set1_epi64x(0x2222222222222222);
364 q2 = q[2] & _mm_set1_epi64x(0x4444444444444444);
365 q3 = q[3] & _mm_set1_epi64x(0x8888888888888888);
366 q4 = _mm_shuffle_epi32(q0, 0x0e);
367 q5 = _mm_shuffle_epi32(q1, 0x0e);
368 q6 = _mm_shuffle_epi32(q2, 0x0e);
369 q7 = _mm_shuffle_epi32(q3, 0x0e);
370 _mm_storeu_si64(&comp_skey[j + 0], q0 | q1 | q2 | q3);
371 _mm_storeu_si64(&comp_skey[j + 1], q4 | q5 | q6 | q7);
372 }
373 return num_rounds;
374 }
375
376 void
aes_sse2_skey_expand(uint64_t * skey,unsigned num_rounds,const uint64_t * comp_skey)377 aes_sse2_skey_expand(uint64_t *skey,
378 unsigned num_rounds, const uint64_t *comp_skey)
379 {
380 unsigned u, v, n;
381
382 n = (num_rounds + 1) << 1;
383 for (u = 0, v = 0; u < n; u ++, v += 4) {
384 __m128i x0, x1, x2, x3;
385
386 x0 = x1 = x2 = x3 = _mm_loadu_si64(&comp_skey[u]);
387 x0 &= 0x1111111111111111;
388 x1 &= 0x2222222222222222;
389 x2 &= 0x4444444444444444;
390 x3 &= 0x8888888888888888;
391 x1 = _mm_srli_epi64(x1, 1);
392 x2 = _mm_srli_epi64(x2, 2);
393 x3 = _mm_srli_epi64(x3, 3);
394 x0 = _mm_sub_epi64(_mm_slli_epi64(x0, 4), x0);
395 x1 = _mm_sub_epi64(_mm_slli_epi64(x1, 4), x1);
396 x2 = _mm_sub_epi64(_mm_slli_epi64(x2, 4), x2);
397 x3 = _mm_sub_epi64(_mm_slli_epi64(x3, 4), x3);
398 _mm_storeu_si64(&skey[v + 0], x0);
399 _mm_storeu_si64(&skey[v + 1], x1);
400 _mm_storeu_si64(&skey[v + 2], x2);
401 _mm_storeu_si64(&skey[v + 3], x3);
402 }
403 }
404