1 /* $NetBSD: aes_ni.c,v 1.5 2020/09/05 07:45:44 maxv Exp $ */
2
3 /*-
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(1, "$NetBSD: aes_ni.c,v 1.5 2020/09/05 07:45:44 maxv Exp $");
31
32 #ifdef _KERNEL
33 #include <sys/types.h>
34 #include <sys/systm.h>
35 #else
36 #include <assert.h>
37 #include <err.h>
38 #include <stdint.h>
39 #include <string.h>
40 #define KASSERT assert
41 #define panic(fmt, args...) err(1, fmt, args)
42 #endif
43
44 #include <crypto/aes/aes.h>
45 #include <crypto/aes/aes_impl.h>
46 #include <crypto/aes/arch/x86/aes_ni.h>
47
48 #ifdef _KERNEL
49 #include <x86/cpuvar.h>
50 #include <x86/fpu.h>
51 #include <x86/specialreg.h>
52 #else
53 #include <cpuid.h>
54 #define fpu_kern_enter() ((void)0)
55 #define fpu_kern_leave() ((void)0)
56 #endif
57
58 static void
aesni_setenckey(struct aesenc * enc,const uint8_t key[static16],uint32_t nrounds)59 aesni_setenckey(struct aesenc *enc, const uint8_t key[static 16],
60 uint32_t nrounds)
61 {
62
63 switch (nrounds) {
64 case 10:
65 aesni_setenckey128(enc, key);
66 break;
67 case 12:
68 aesni_setenckey192(enc, key);
69 break;
70 case 14:
71 aesni_setenckey256(enc, key);
72 break;
73 default:
74 panic("invalid AES rounds: %u", nrounds);
75 }
76 }
77
78 static void
aesni_setenckey_impl(struct aesenc * enc,const uint8_t key[static16],uint32_t nrounds)79 aesni_setenckey_impl(struct aesenc *enc, const uint8_t key[static 16],
80 uint32_t nrounds)
81 {
82
83 fpu_kern_enter();
84 aesni_setenckey(enc, key, nrounds);
85 fpu_kern_leave();
86 }
87
88 static void
aesni_setdeckey_impl(struct aesdec * dec,const uint8_t key[static16],uint32_t nrounds)89 aesni_setdeckey_impl(struct aesdec *dec, const uint8_t key[static 16],
90 uint32_t nrounds)
91 {
92 struct aesenc enc;
93
94 fpu_kern_enter();
95 aesni_setenckey(&enc, key, nrounds);
96 aesni_enctodec(&enc, dec, nrounds);
97 fpu_kern_leave();
98
99 explicit_memset(&enc, 0, sizeof enc);
100 }
101
102 static void
aesni_enc_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],uint32_t nrounds)103 aesni_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
104 uint8_t out[static 16], uint32_t nrounds)
105 {
106
107 fpu_kern_enter();
108 aesni_enc(enc, in, out, nrounds);
109 fpu_kern_leave();
110 }
111
112 static void
aesni_dec_impl(const struct aesdec * dec,const uint8_t in[static16],uint8_t out[static16],uint32_t nrounds)113 aesni_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
114 uint8_t out[static 16], uint32_t nrounds)
115 {
116
117 fpu_kern_enter();
118 aesni_dec(dec, in, out, nrounds);
119 fpu_kern_leave();
120 }
121
122 static void
aesni_cbc_enc_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)123 aesni_cbc_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
124 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
125 uint32_t nrounds)
126 {
127
128 KASSERT(nbytes % 16 == 0);
129
130 fpu_kern_enter();
131 aesni_cbc_enc(enc, in, out, nbytes, iv, nrounds);
132 fpu_kern_leave();
133 }
134
135 static void
aesni_cbc_dec_impl(const struct aesdec * dec,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)136 aesni_cbc_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
137 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
138 uint32_t nrounds)
139 {
140
141 KASSERT(nbytes % 16 == 0);
142
143 fpu_kern_enter();
144
145 if (nbytes % 128) {
146 aesni_cbc_dec1(dec, in, out, nbytes % 128, iv, nrounds);
147 in += nbytes % 128;
148 out += nbytes % 128;
149 nbytes -= nbytes % 128;
150 }
151
152 KASSERT(nbytes % 128 == 0);
153 if (nbytes)
154 aesni_cbc_dec8(dec, in, out, nbytes, iv, nrounds);
155
156 fpu_kern_leave();
157 }
158
159 static void
aesni_xts_enc_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)160 aesni_xts_enc_impl(const struct aesenc *enc, const uint8_t in[static 16],
161 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
162 uint32_t nrounds)
163 {
164
165 KASSERT(nbytes % 16 == 0);
166
167 fpu_kern_enter();
168
169 if (nbytes % 128) {
170 aesni_xts_enc1(enc, in, out, nbytes % 128, iv, nrounds);
171 in += nbytes % 128;
172 out += nbytes % 128;
173 nbytes -= nbytes % 128;
174 }
175
176 KASSERT(nbytes % 128 == 0);
177 if (nbytes)
178 aesni_xts_enc8(enc, in, out, nbytes, iv, nrounds);
179
180 fpu_kern_leave();
181 }
182
183 static void
aesni_xts_dec_impl(const struct aesdec * dec,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t iv[static16],uint32_t nrounds)184 aesni_xts_dec_impl(const struct aesdec *dec, const uint8_t in[static 16],
185 uint8_t out[static 16], size_t nbytes, uint8_t iv[static 16],
186 uint32_t nrounds)
187 {
188
189 KASSERT(nbytes % 16 == 0);
190
191 fpu_kern_enter();
192
193 if (nbytes % 128) {
194 aesni_xts_dec1(dec, in, out, nbytes % 128, iv, nrounds);
195 in += nbytes % 128;
196 out += nbytes % 128;
197 nbytes -= nbytes % 128;
198 }
199
200 KASSERT(nbytes % 128 == 0);
201 if (nbytes)
202 aesni_xts_dec8(dec, in, out, nbytes, iv, nrounds);
203
204 fpu_kern_leave();
205 }
206
207 static void
aesni_cbcmac_update1_impl(const struct aesenc * enc,const uint8_t in[static16],size_t nbytes,uint8_t auth[static16],uint32_t nrounds)208 aesni_cbcmac_update1_impl(const struct aesenc *enc,
209 const uint8_t in[static 16], size_t nbytes, uint8_t auth[static 16],
210 uint32_t nrounds)
211 {
212
213 KASSERT(nbytes);
214 KASSERT(nbytes % 16 == 0);
215
216 fpu_kern_enter();
217 aesni_cbcmac_update1(enc, in, nbytes, auth, nrounds);
218 fpu_kern_leave();
219 }
220
221 static void
aesni_ccm_enc1_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t authctr[static32],uint32_t nrounds)222 aesni_ccm_enc1_impl(const struct aesenc *enc, const uint8_t in[static 16],
223 uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
224 uint32_t nrounds)
225 {
226
227 KASSERT(nbytes);
228 KASSERT(nbytes % 16 == 0);
229
230 fpu_kern_enter();
231 aesni_ccm_enc1(enc, in, out, nbytes, authctr, nrounds);
232 fpu_kern_leave();
233 }
234
235 static void
aesni_ccm_dec1_impl(const struct aesenc * enc,const uint8_t in[static16],uint8_t out[static16],size_t nbytes,uint8_t authctr[static32],uint32_t nrounds)236 aesni_ccm_dec1_impl(const struct aesenc *enc, const uint8_t in[static 16],
237 uint8_t out[static 16], size_t nbytes, uint8_t authctr[static 32],
238 uint32_t nrounds)
239 {
240
241 KASSERT(nbytes);
242 KASSERT(nbytes % 16 == 0);
243
244 fpu_kern_enter();
245 aesni_ccm_dec1(enc, in, out, nbytes, authctr, nrounds);
246 fpu_kern_leave();
247 }
248
249 static int
aesni_xts_update_selftest(void)250 aesni_xts_update_selftest(void)
251 {
252 static const struct {
253 uint8_t in[16], out[16];
254 } cases[] = {
255 {{1}, {2}},
256 {{0,0,0,0x80}, {0,0,0,0,1}},
257 {{0,0,0,0,0,0,0,0x80}, {0,0,0,0,0,0,0,0,1}},
258 {{0,0,0,0x80,0,0,0,0x80}, {0,0,0,0,1,0,0,0,1}},
259 {{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0x80}, {0x87}},
260 {{0,0,0,0,0,0,0,0x80,0,0,0,0,0,0,0,0x80},
261 {0x87,0,0,0,0,0,0,0,1}},
262 {{0,0,0,0x80,0,0,0,0,0,0,0,0,0,0,0,0x80}, {0x87,0,0,0,1}},
263 {{0,0,0,0x80,0,0,0,0x80,0,0,0,0,0,0,0,0x80},
264 {0x87,0,0,0,1,0,0,0,1}},
265 };
266 unsigned i;
267 uint8_t tweak[16];
268
269 for (i = 0; i < sizeof(cases)/sizeof(cases[0]); i++) {
270 aesni_xts_update(cases[i].in, tweak);
271 if (memcmp(tweak, cases[i].out, 16))
272 return -1;
273 }
274
275 /* Success! */
276 return 0;
277 }
278
279 static int
aesni_probe(void)280 aesni_probe(void)
281 {
282 int result = 0;
283
284 /* Verify that the CPU supports AES-NI. */
285 #ifdef _KERNEL
286 if ((cpu_feature[1] & CPUID2_AESNI) == 0)
287 return -1;
288 #else
289 unsigned eax, ebx, ecx, edx;
290 if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx))
291 return -1;
292 if ((ecx & bit_AES) == 0)
293 return -1;
294 #endif
295
296 fpu_kern_enter();
297
298 /* Verify that our XTS tweak update logic works. */
299 if (aesni_xts_update_selftest())
300 result = -1;
301
302 fpu_kern_leave();
303
304 return result;
305 }
306
307 struct aes_impl aes_ni_impl = {
308 .ai_name = "Intel AES-NI",
309 .ai_probe = aesni_probe,
310 .ai_setenckey = aesni_setenckey_impl,
311 .ai_setdeckey = aesni_setdeckey_impl,
312 .ai_enc = aesni_enc_impl,
313 .ai_dec = aesni_dec_impl,
314 .ai_cbc_enc = aesni_cbc_enc_impl,
315 .ai_cbc_dec = aesni_cbc_dec_impl,
316 .ai_xts_enc = aesni_xts_enc_impl,
317 .ai_xts_dec = aesni_xts_dec_impl,
318 .ai_cbcmac_update1 = aesni_cbcmac_update1_impl,
319 .ai_ccm_enc1 = aesni_ccm_enc1_impl,
320 .ai_ccm_dec1 = aesni_ccm_dec1_impl,
321 };
322