1 /* $OpenBSD: octcrypto.c,v 1.8 2021/10/24 10:26:22 patrick Exp $ */
2
3 /*
4 * Copyright (c) 2018 Visa Hankala
5 *
6 * Permission to use, copy, modify, and/or distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 /*
20 * Driver for the OCTEON cryptographic unit.
21 */
22
23 #include <sys/param.h>
24 #include <sys/systm.h>
25 #include <sys/atomic.h>
26 #include <sys/malloc.h>
27 #include <sys/mbuf.h>
28 #include <sys/pool.h>
29 #include <sys/smr.h>
30 #include <sys/tree.h>
31
32 #include <crypto/cryptodev.h>
33 #include <crypto/cryptosoft.h>
34 #include <crypto/xform.h>
35
36 #include <mips64/mips_cpu.h>
37
38 #include <machine/octeonvar.h>
39
40 /* Maximum number of dwords in hash IV. */
41 #define MAX_IVNW 8
42
43 /* Number of dwords needed to cover `n' bytes. */
44 #define ndwords(n) (roundup(n, 8) / (8))
45
46 struct octcrypto_softc;
47
48 struct octcrypto_hmac {
49 void (*transform)(const void *, size_t);
50 void (*get_iv)(uint64_t *);
51 void (*set_iv)(const uint64_t *);
52 void (*clear)(void);
53 uint16_t blocklen;
54 uint16_t taglen;
55 uint16_t countlen;
56 };
57
58 struct octcrypto_session {
59 uint32_t ses_sid; /* RB key, keep first */
60 RBT_ENTRY(octrcypto_session)
61 ses_entry;
62 struct octcrypto_softc *ses_sc;
63 struct smr_entry ses_smr;
64
65 /* AES parameters */
66 uint64_t ses_key[4];
67 int ses_klen;
68 uint8_t ses_nonce[AESCTR_NONCESIZE];
69
70 /* HMAC parameters */
71 const struct octcrypto_hmac
72 *ses_hmac;
73 uint64_t ses_iiv[MAX_IVNW]; /* HMAC inner IV */
74 uint64_t ses_oiv[MAX_IVNW]; /* HMAC outer IV */
75
76 /* GHASH parameters */
77 uint64_t ses_ghkey[2];
78
79 struct swcr_data *ses_swd;
80 };
81
82 struct octcrypto_cpu {
83 uint8_t *pcpu_buf;
84 size_t pcpu_buflen;
85 };
86
87 struct octcrypto_softc {
88 struct device sc_dev;
89 int32_t sc_cid;
90 uint32_t sc_sid;
91 struct mutex sc_mtx;
92 RBT_HEAD(octcrypto_tree, octcrypto_session)
93 sc_sessions;
94 struct octcrypto_cpu sc_cpu[MAXCPUS];
95 };
96
97 int octcrypto_match(struct device *, void *, void *);
98 void octcrypto_attach(struct device *, struct device *, void *);
99
100 int octcrypto_newsession(uint32_t *, struct cryptoini *);
101 int octcrypto_freesession(uint64_t);
102 int octcrypto_process(struct cryptop *);
103
104 struct octcrypto_session *
105 octcrypto_get(struct octcrypto_softc *, uint32_t);
106 void octcrypto_free(struct octcrypto_session *);
107 void octcrypto_free_smr(void *);
108
109 void octcrypto_hmac(struct cryptodesc *, uint8_t *, size_t,
110 struct octcrypto_session *, uint64_t *);
111 int octcrypto_authenc_gmac(struct cryptop *, struct cryptodesc *,
112 struct cryptodesc *, struct octcrypto_session *);
113 int octcrypto_authenc_hmac(struct cryptop *, struct cryptodesc *,
114 struct cryptodesc *, struct octcrypto_session *);
115 int octcrypto_swauth(struct cryptop *, struct cryptodesc *,
116 struct swcr_data *, uint8_t *);
117
118 void octcrypto_ghash_update_md(GHASH_CTX *, uint8_t *, size_t);
119
120 void octcrypto_aes_clear(void);
121 void octcrypto_aes_cbc_dec(void *, size_t, const void *);
122 void octcrypto_aes_cbc_enc(void *, size_t, const void *);
123 void octcrypto_aes_ctr_enc(void *, size_t, const void *);
124 void octcrypto_aes_enc(uint64_t *);
125 void octcrypto_aes_set_key(const uint64_t *, int);
126
127 void octcrypto_ghash_finish(uint64_t *);
128 void octcrypto_ghash_init(const uint64_t *, const uint64_t *);
129 void octcrypto_ghash_update(const void *, size_t);
130
131 void octcrypto_hash_md5(const void *, size_t);
132 void octcrypto_hash_sha1(const void *, size_t);
133 void octcrypto_hash_sha256(const void *, size_t);
134 void octcrypto_hash_sha512(const void *, size_t);
135 void octcrypto_hash_clearn(void);
136 void octcrypto_hash_clearw(void);
137 void octcrypto_hash_get_ivn(uint64_t *);
138 void octcrypto_hash_get_ivw(uint64_t *);
139 void octcrypto_hash_set_ivn(const uint64_t *);
140 void octcrypto_hash_set_ivw(const uint64_t *);
141
142 const struct cfattach octcrypto_ca = {
143 sizeof(struct octcrypto_softc), octcrypto_match, octcrypto_attach
144 };
145
146 struct cfdriver octcrypto_cd = {
147 NULL, "octcrypto", DV_DULL
148 };
149
150 static const struct octcrypto_hmac hmac_md5_96 = {
151 .transform = octcrypto_hash_md5,
152 .get_iv = octcrypto_hash_get_ivn,
153 .set_iv = octcrypto_hash_set_ivn,
154 .clear = octcrypto_hash_clearn,
155 .blocklen = 64,
156 .taglen = 12,
157 .countlen = 8
158 };
159
160 static const struct octcrypto_hmac hmac_sha1_96 = {
161 .transform = octcrypto_hash_sha1,
162 .get_iv = octcrypto_hash_get_ivn,
163 .set_iv = octcrypto_hash_set_ivn,
164 .clear = octcrypto_hash_clearn,
165 .blocklen = 64,
166 .taglen = 12,
167 .countlen = 8
168 };
169
170 static const struct octcrypto_hmac hmac_sha2_256_128 = {
171 .transform = octcrypto_hash_sha256,
172 .get_iv = octcrypto_hash_get_ivn,
173 .set_iv = octcrypto_hash_set_ivn,
174 .clear = octcrypto_hash_clearn,
175 .blocklen = 64,
176 .taglen = 16,
177 .countlen = 8
178 };
179
180 static const struct octcrypto_hmac hmac_sha2_384_192 = {
181 .transform = octcrypto_hash_sha512,
182 .get_iv = octcrypto_hash_get_ivw,
183 .set_iv = octcrypto_hash_set_ivw,
184 .clear = octcrypto_hash_clearw,
185 .blocklen = 128,
186 .taglen = 24,
187 .countlen = 16
188 };
189
190 static const struct octcrypto_hmac hmac_sha2_512_256 = {
191 .transform = octcrypto_hash_sha512,
192 .get_iv = octcrypto_hash_get_ivw,
193 .set_iv = octcrypto_hash_set_ivw,
194 .clear = octcrypto_hash_clearw,
195 .blocklen = 128,
196 .taglen = 32,
197 .countlen = 16
198 };
199
200 static struct pool octcryptopl;
201 static struct octcrypto_softc *octcrypto_sc;
202
203 static inline int
octcrypto_cmp(const struct octcrypto_session * a,const struct octcrypto_session * b)204 octcrypto_cmp(const struct octcrypto_session *a,
205 const struct octcrypto_session *b)
206 {
207 if (a->ses_sid < b->ses_sid)
208 return -1;
209 if (a->ses_sid > b->ses_sid)
210 return 1;
211 return 0;
212 }
213
214 RBT_PROTOTYPE(octcrypto_tree, octcrypto_session, sess_entry, octcrypto_cmp);
215 RBT_GENERATE(octcrypto_tree, octcrypto_session, ses_entry, octcrypto_cmp);
216
217 static inline void
cop2_enable(void)218 cop2_enable(void)
219 {
220 setsr(getsr() | SR_COP_2_BIT);
221 }
222
223 static inline void
cop2_disable(void)224 cop2_disable(void)
225 {
226 setsr(getsr() & ~SR_COP_2_BIT);
227 }
228
229 int
octcrypto_match(struct device * parent,void * match,void * aux)230 octcrypto_match(struct device *parent, void *match, void *aux)
231 {
232 return 1;
233 }
234
235 void
octcrypto_attach(struct device * parent,struct device * self,void * aux)236 octcrypto_attach(struct device *parent, struct device *self, void *aux)
237 {
238 int algs[CRYPTO_ALGORITHM_MAX + 1];
239 struct octcrypto_softc *sc = (struct octcrypto_softc *)self;
240
241 pool_init(&octcryptopl, sizeof(struct octcrypto_session), 0, IPL_VM, 0,
242 "octcrypto", NULL);
243 pool_setlowat(&octcryptopl, 2);
244
245 mtx_init(&sc->sc_mtx, IPL_VM);
246 RBT_INIT(octcrypto_tree, &sc->sc_sessions);
247
248 sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_MPSAFE);
249 if (sc->sc_cid < 0) {
250 printf(": could not get driver id\n");
251 return;
252 }
253
254 printf("\n");
255
256 memset(algs, 0, sizeof(algs));
257
258 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
259 algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED;
260 algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED;
261
262 algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
263 algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
264 algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
265 algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
266
267 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
268 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
269 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
270 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
271 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
272
273 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
274
275 algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED;
276
277 octcrypto_sc = sc;
278
279 crypto_register(sc->sc_cid, algs, octcrypto_newsession,
280 octcrypto_freesession, octcrypto_process);
281
282 ghash_update = octcrypto_ghash_update_md;
283 }
284
285 struct octcrypto_session *
octcrypto_get(struct octcrypto_softc * sc,uint32_t sid)286 octcrypto_get(struct octcrypto_softc *sc, uint32_t sid)
287 {
288 struct octcrypto_session *ses;
289
290 SMR_ASSERT_CRITICAL();
291
292 mtx_enter(&sc->sc_mtx);
293 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions,
294 (struct octcrypto_session *)&sid);
295 mtx_leave(&sc->sc_mtx);
296 return ses;
297 }
298
299 void
octcrypto_free(struct octcrypto_session * ses)300 octcrypto_free(struct octcrypto_session *ses)
301 {
302 const struct auth_hash *axf;
303 struct swcr_data *swd;
304
305 if (ses->ses_swd != NULL) {
306 swd = ses->ses_swd;
307 axf = swd->sw_axf;
308
309 if (swd->sw_ictx != NULL) {
310 explicit_bzero(swd->sw_ictx, axf->ctxsize);
311 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize);
312 }
313 if (swd->sw_octx != NULL) {
314 explicit_bzero(swd->sw_octx, axf->ctxsize);
315 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize);
316 }
317 free(swd, M_CRYPTO_DATA, sizeof(*swd));
318 }
319
320 explicit_bzero(ses, sizeof(*ses));
321 pool_put(&octcryptopl, ses);
322 }
323
324 void
octcrypto_free_smr(void * arg)325 octcrypto_free_smr(void *arg)
326 {
327 struct octcrypto_session *ses = arg;
328
329 octcrypto_free(ses);
330 }
331
332 int
octcrypto_newsession(uint32_t * sidp,struct cryptoini * cri)333 octcrypto_newsession(uint32_t *sidp, struct cryptoini *cri)
334 {
335 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)];
336 const struct auth_hash *axf;
337 struct cryptoini *c;
338 const struct octcrypto_hmac *hmac = NULL;
339 struct octcrypto_softc *sc = octcrypto_sc;
340 struct octcrypto_session *ses = NULL;
341 struct swcr_data *swd;
342 uint8_t *bptr;
343 size_t klen;
344 int i;
345 uint32_t sid;
346
347 if (sidp == NULL || cri == NULL)
348 return EINVAL;
349
350 ses = pool_get(&octcryptopl, PR_NOWAIT | PR_ZERO);
351 if (ses == NULL)
352 return ENOMEM;
353 ses->ses_sc = sc;
354 smr_init(&ses->ses_smr);
355
356 for (c = cri; c != NULL; c = c->cri_next) {
357 switch (c->cri_alg) {
358 case CRYPTO_AES_CBC:
359 ses->ses_klen = c->cri_klen / 8;
360 memcpy(ses->ses_key, c->cri_key, ses->ses_klen);
361 break;
362
363 case CRYPTO_AES_CTR:
364 case CRYPTO_AES_GCM_16:
365 case CRYPTO_AES_GMAC:
366 ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE;
367 memcpy(ses->ses_key, c->cri_key, ses->ses_klen);
368 memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen,
369 AESCTR_NONCESIZE);
370 break;
371
372 case CRYPTO_AES_128_GMAC:
373 case CRYPTO_AES_192_GMAC:
374 case CRYPTO_AES_256_GMAC:
375 cop2_enable();
376 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen);
377 octcrypto_aes_enc(ses->ses_ghkey);
378 octcrypto_aes_clear();
379 cop2_disable();
380 break;
381
382 case CRYPTO_MD5_HMAC:
383 ses->ses_iiv[0] = 0x0123456789abcdefULL;
384 ses->ses_iiv[1] = 0xfedcba9876543210ULL;
385 ses->ses_hmac = &hmac_md5_96;
386 goto hwauthcommon;
387
388 case CRYPTO_SHA1_HMAC:
389 ses->ses_iiv[0] = 0x67452301efcdab89ULL;
390 ses->ses_iiv[1] = 0x98badcfe10325476ULL;
391 ses->ses_iiv[2] = 0xc3d2e1f000000000ULL;
392 ses->ses_hmac = &hmac_sha1_96;
393 goto hwauthcommon;
394
395 case CRYPTO_SHA2_256_HMAC:
396 ses->ses_iiv[0] = 0x6a09e667bb67ae85ULL;
397 ses->ses_iiv[1] = 0x3c6ef372a54ff53aULL;
398 ses->ses_iiv[2] = 0x510e527f9b05688cULL;
399 ses->ses_iiv[3] = 0x1f83d9ab5be0cd19ULL;
400 ses->ses_hmac = &hmac_sha2_256_128;
401 goto hwauthcommon;
402
403 case CRYPTO_SHA2_384_HMAC:
404 ses->ses_iiv[0] = 0xcbbb9d5dc1059ed8ULL;
405 ses->ses_iiv[1] = 0x629a292a367cd507ULL;
406 ses->ses_iiv[2] = 0x9159015a3070dd17ULL;
407 ses->ses_iiv[3] = 0x152fecd8f70e5939ULL;
408 ses->ses_iiv[4] = 0x67332667ffc00b31ULL;
409 ses->ses_iiv[5] = 0x8eb44a8768581511ULL;
410 ses->ses_iiv[6] = 0xdb0c2e0d64f98fa7ULL;
411 ses->ses_iiv[7] = 0x47b5481dbefa4fa4ULL;
412 ses->ses_hmac = &hmac_sha2_384_192;
413 goto hwauthcommon;
414
415 case CRYPTO_SHA2_512_HMAC:
416 ses->ses_iiv[0] = 0x6a09e667f3bcc908ULL;
417 ses->ses_iiv[1] = 0xbb67ae8584caa73bULL;
418 ses->ses_iiv[2] = 0x3c6ef372fe94f82bULL;
419 ses->ses_iiv[3] = 0xa54ff53a5f1d36f1ULL;
420 ses->ses_iiv[4] = 0x510e527fade682d1ULL;
421 ses->ses_iiv[5] = 0x9b05688c2b3e6c1fULL;
422 ses->ses_iiv[6] = 0x1f83d9abfb41bd6bULL;
423 ses->ses_iiv[7] = 0x5be0cd19137e2179ULL;
424 ses->ses_hmac = &hmac_sha2_512_256;
425
426 hwauthcommon:
427 memcpy(ses->ses_oiv, ses->ses_iiv,
428 sizeof(uint64_t) * MAX_IVNW);
429
430 bptr = (char *)block;
431 klen = c->cri_klen / 8;
432 hmac = ses->ses_hmac;
433
434 memcpy(bptr, c->cri_key, klen);
435 memset(bptr + klen, 0, hmac->blocklen - klen);
436 for (i = 0; i < hmac->blocklen; i++)
437 bptr[i] ^= HMAC_IPAD_VAL;
438
439 cop2_enable();
440 hmac->set_iv(ses->ses_iiv);
441 hmac->transform(block, hmac->blocklen);
442 hmac->get_iv(ses->ses_iiv);
443
444 for (i = 0; i < hmac->blocklen; i++)
445 bptr[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
446
447 hmac->set_iv(ses->ses_oiv);
448 hmac->transform(block, hmac->blocklen);
449 hmac->get_iv(ses->ses_oiv);
450 hmac->clear();
451 cop2_disable();
452
453 explicit_bzero(block, hmac->blocklen);
454 break;
455
456 case CRYPTO_RIPEMD160_HMAC:
457 axf = &auth_hash_hmac_ripemd_160_96;
458 goto swauthcommon;
459
460 swauthcommon:
461 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA,
462 M_NOWAIT | M_ZERO);
463 if (swd == NULL) {
464 octcrypto_free(ses);
465 return ENOMEM;
466 }
467 ses->ses_swd = swd;
468
469 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
470 M_NOWAIT);
471 if (swd->sw_ictx == NULL) {
472 octcrypto_free(ses);
473 return ENOMEM;
474 }
475
476 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
477 M_NOWAIT);
478 if (swd->sw_octx == NULL) {
479 octcrypto_free(ses);
480 return ENOMEM;
481 }
482
483 for (i = 0; i < c->cri_klen / 8; i++)
484 c->cri_key[i] ^= HMAC_IPAD_VAL;
485
486 axf->Init(swd->sw_ictx);
487 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8);
488 axf->Update(swd->sw_ictx, hmac_ipad_buffer,
489 axf->blocksize - (c->cri_klen / 8));
490
491 for (i = 0; i < c->cri_klen / 8; i++)
492 c->cri_key[i] ^= (HMAC_IPAD_VAL ^
493 HMAC_OPAD_VAL);
494
495 axf->Init(swd->sw_octx);
496 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8);
497 axf->Update(swd->sw_octx, hmac_opad_buffer,
498 axf->blocksize - (c->cri_klen / 8));
499
500 for (i = 0; i < c->cri_klen / 8; i++)
501 c->cri_key[i] ^= HMAC_OPAD_VAL;
502
503 swd->sw_axf = axf;
504 swd->sw_alg = c->cri_alg;
505
506 break;
507
508 case CRYPTO_ESN:
509 /* nothing to do */
510 break;
511
512 default:
513 octcrypto_free(ses);
514 return EINVAL;
515 }
516 }
517
518 mtx_enter(&sc->sc_mtx);
519 /* Find a free session ID. Assume there is one. */
520 do {
521 sc->sc_sid++;
522 if (sc->sc_sid == 0)
523 sc->sc_sid = 1;
524 sid = sc->sc_sid;
525 } while (RBT_FIND(octcrypto_tree, &sc->sc_sessions,
526 (struct octcrypto_session *)&sid) != NULL);
527 ses->ses_sid = sid;
528 RBT_INSERT(octcrypto_tree, &sc->sc_sessions, ses);
529 mtx_leave(&sc->sc_mtx);
530
531 *sidp = ses->ses_sid;
532 return 0;
533 }
534
535 int
octcrypto_freesession(uint64_t tid)536 octcrypto_freesession(uint64_t tid)
537 {
538 struct octcrypto_softc *sc = octcrypto_sc;
539 struct octcrypto_session *ses;
540 uint32_t sid = (uint32_t)tid;
541
542 mtx_enter(&sc->sc_mtx);
543 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions,
544 (struct octcrypto_session *)&sid);
545 if (ses != NULL)
546 RBT_REMOVE(octcrypto_tree, &sc->sc_sessions, ses);
547 mtx_leave(&sc->sc_mtx);
548
549 if (ses == NULL)
550 return EINVAL;
551
552 smr_call(&ses->ses_smr, octcrypto_free_smr, ses);
553
554 return 0;
555 }
556
557 enum {
558 ALG_UNHANDLED,
559 ALG_AES,
560 ALG_AES_GHASH,
561 ALG_GMAC,
562 ALG_HMAC
563 };
564
565 static int
alg_class(int alg)566 alg_class(int alg)
567 {
568 switch (alg) {
569 case CRYPTO_AES_CBC:
570 case CRYPTO_AES_CTR:
571 return ALG_AES;
572 case CRYPTO_AES_GCM_16:
573 case CRYPTO_AES_GMAC:
574 return ALG_AES_GHASH;
575 case CRYPTO_AES_128_GMAC:
576 case CRYPTO_AES_192_GMAC:
577 case CRYPTO_AES_256_GMAC:
578 return ALG_GMAC;
579 case CRYPTO_MD5_HMAC:
580 case CRYPTO_SHA1_HMAC:
581 case CRYPTO_SHA2_256_HMAC:
582 case CRYPTO_SHA2_384_HMAC:
583 case CRYPTO_SHA2_512_HMAC:
584 return ALG_HMAC;
585 default:
586 return ALG_UNHANDLED;
587 }
588 }
589
590 int
octcrypto_process(struct cryptop * crp)591 octcrypto_process(struct cryptop *crp)
592 {
593 struct cryptodesc *crd, *crd2;
594 struct octcrypto_softc *sc = octcrypto_sc;
595 struct octcrypto_session *ses = NULL;
596 int alg, alg2;
597 int error = 0;
598 int i;
599
600 KASSERT(crp->crp_ndesc >= 1);
601
602 smr_read_enter();
603 ses = octcrypto_get(sc, (uint32_t)crp->crp_sid);
604 if (ses == NULL) {
605 error = EINVAL;
606 goto out;
607 }
608
609 if (crp->crp_ndesc == 2) {
610 crd = &crp->crp_desc[0];
611 crd2 = &crp->crp_desc[1];
612 alg = alg_class(crd->crd_alg);
613 alg2 = alg_class(crd2->crd_alg);
614
615 if ((alg == ALG_AES) && (alg2 == ALG_HMAC)) {
616 error = octcrypto_authenc_hmac(crp, crd, crd2, ses);
617 goto out;
618 } else if ((alg2 == ALG_AES) && (alg == ALG_HMAC)) {
619 error = octcrypto_authenc_hmac(crp, crd2, crd, ses);
620 goto out;
621 } else if ((alg == ALG_AES_GHASH) && (alg2 == ALG_GMAC)) {
622 error = octcrypto_authenc_gmac(crp, crd, crd2, ses);
623 goto out;
624 } else if ((alg2 == ALG_AES_GHASH) && (alg == ALG_GMAC)) {
625 error = octcrypto_authenc_gmac(crp, crd2, crd, ses);
626 goto out;
627 }
628 }
629
630 for (i = 0; i < crp->crp_ndesc; i++) {
631 crd = &crp->crp_desc[i];
632 switch (crd->crd_alg) {
633 case CRYPTO_AES_CBC:
634 case CRYPTO_AES_CTR:
635 error = octcrypto_authenc_hmac(crp, crd, NULL, ses);
636 break;
637
638 case CRYPTO_MD5_HMAC:
639 case CRYPTO_SHA1_HMAC:
640 case CRYPTO_SHA2_256_HMAC:
641 case CRYPTO_SHA2_384_HMAC:
642 case CRYPTO_SHA2_512_HMAC:
643 error = octcrypto_authenc_hmac(crp, NULL, crd, ses);
644 break;
645
646 case CRYPTO_RIPEMD160_HMAC:
647 error = octcrypto_swauth(crp, crd, ses->ses_swd,
648 crp->crp_buf);
649 break;
650
651 default:
652 error = EINVAL;
653 break;
654 }
655 }
656
657 out:
658 smr_read_leave();
659 return error;
660 }
661
662 int
octcrypto_swauth(struct cryptop * crp,struct cryptodesc * crd,struct swcr_data * sw,uint8_t * buf)663 octcrypto_swauth(struct cryptop *crp, struct cryptodesc *crd,
664 struct swcr_data *sw, uint8_t *buf)
665 {
666 int type;
667
668 if (crp->crp_flags & CRYPTO_F_IMBUF)
669 type = CRYPTO_BUF_MBUF;
670 else
671 type = CRYPTO_BUF_IOV;
672
673 return swcr_authcompute(crp, crd, sw, buf, type);
674 }
675
676 int
octcrypto_authenc_gmac(struct cryptop * crp,struct cryptodesc * crde,struct cryptodesc * crda,struct octcrypto_session * ses)677 octcrypto_authenc_gmac(struct cryptop *crp, struct cryptodesc *crde,
678 struct cryptodesc *crda, struct octcrypto_session *ses)
679 {
680 uint64_t block[ndwords(AESCTR_BLOCKSIZE)];
681 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)];
682 uint64_t iv[ndwords(AESCTR_BLOCKSIZE)];
683 uint64_t tag[ndwords(GMAC_BLOCK_LEN)];
684 uint8_t *buf;
685 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()];
686 size_t aadlen;
687 size_t ivlen = 8;
688 size_t rlen;
689 int error = 0;
690 unsigned int iskip = 0;
691 unsigned int oskip = 0;
692
693 KASSERT(crda != NULL);
694 KASSERT(crde != NULL);
695
696 rlen = roundup(crde->crd_len, AESCTR_BLOCKSIZE);
697 if (rlen > pcpu->pcpu_buflen) {
698 if (pcpu->pcpu_buf != NULL) {
699 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen);
700 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen);
701 }
702 pcpu->pcpu_buflen = 0;
703 pcpu->pcpu_buf = malloc(rlen, M_DEVBUF, M_NOWAIT | M_ZERO);
704 if (pcpu->pcpu_buf == NULL)
705 return ENOMEM;
706 pcpu->pcpu_buflen = rlen;
707 }
708 buf = pcpu->pcpu_buf;
709
710 /* Prepare the IV. */
711 if (crde->crd_flags & CRD_F_ENCRYPT) {
712 if (crde->crd_flags & CRD_F_IV_EXPLICIT)
713 memcpy(iv, crde->crd_iv, ivlen);
714 else
715 arc4random_buf(iv, ivlen);
716
717 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) {
718 if (crp->crp_flags & CRYPTO_F_IMBUF) {
719 if (m_copyback((struct mbuf *)crp->crp_buf,
720 crde->crd_inject, ivlen, (uint8_t *)iv,
721 M_NOWAIT)) {
722 error = ENOMEM;
723 goto out;
724 }
725 } else {
726 cuio_copyback((struct uio *)crp->crp_buf,
727 crde->crd_inject, ivlen, (uint8_t *)iv);
728 }
729 }
730 } else {
731 if (crde->crd_flags & CRD_F_IV_EXPLICIT) {
732 memcpy(iv, crde->crd_iv, ivlen);
733 } else {
734 if (crp->crp_flags & CRYPTO_F_IMBUF)
735 m_copydata((struct mbuf *)crp->crp_buf,
736 crde->crd_inject, ivlen, iv);
737 else
738 cuio_copydata((struct uio *)crp->crp_buf,
739 crde->crd_inject, ivlen, (uint8_t *)iv);
740 }
741 }
742
743 memset(icb, 0, sizeof(icb));
744 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE);
745 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE);
746 ((uint8_t *)icb)[AESCTR_BLOCKSIZE - 1] = 1;
747
748 /* Prepare the AAD. */
749 aadlen = crda->crd_len;
750 if (crda->crd_flags & CRD_F_ESN) {
751 aadlen += 4;
752 if (crp->crp_flags & CRYPTO_F_IMBUF)
753 m_copydata((struct mbuf *)crp->crp_buf,
754 crda->crd_skip, 4, buf);
755 else
756 cuio_copydata((struct uio *)crp->crp_buf,
757 crda->crd_skip, 4, buf);
758 memcpy(buf + 4, crda->crd_esn, 4);
759 iskip = 4;
760 oskip = 8;
761 }
762 if (crp->crp_flags & CRYPTO_F_IMBUF)
763 m_copydata((struct mbuf *)crp->crp_buf,
764 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip);
765 else
766 cuio_copydata((struct uio *)crp->crp_buf,
767 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip);
768
769 cop2_enable();
770 octcrypto_ghash_init(ses->ses_ghkey, NULL);
771 octcrypto_ghash_update(buf, roundup(aadlen, GMAC_BLOCK_LEN));
772 cop2_disable();
773
774 memset(buf, 0, aadlen);
775
776 /* Copy input to the working buffer. */
777 if (crp->crp_flags & CRYPTO_F_IMBUF)
778 m_copydata((struct mbuf *)crp->crp_buf, crde->crd_skip,
779 crde->crd_len, buf);
780 else
781 cuio_copydata((struct uio *)crp->crp_buf, crde->crd_skip,
782 crde->crd_len, buf);
783
784 cop2_enable();
785 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen);
786
787 switch (crde->crd_alg) {
788 case CRYPTO_AES_GCM_16:
789 if (crde->crd_flags & CRD_F_ENCRYPT) {
790 octcrypto_aes_ctr_enc(buf, rlen, icb);
791 memset(buf + crde->crd_len, 0, rlen - crde->crd_len);
792 octcrypto_ghash_update(buf, rlen);
793 } else {
794 octcrypto_ghash_update(buf, rlen);
795 octcrypto_aes_ctr_enc(buf, rlen, icb);
796 }
797 break;
798
799 case CRYPTO_AES_GMAC:
800 octcrypto_ghash_update(buf, rlen);
801 break;
802 }
803
804 block[0] = htobe64(aadlen * 8);
805 block[1] = htobe64(crde->crd_len * 8);
806 octcrypto_ghash_update(block, GMAC_BLOCK_LEN);
807 octcrypto_ghash_finish(tag);
808
809 block[0] = icb[0];
810 block[1] = icb[1];
811 octcrypto_aes_enc(block);
812 tag[0] ^= block[0];
813 tag[1] ^= block[1];
814
815 octcrypto_aes_clear();
816 cop2_disable();
817
818 /* Copy back the output. */
819 if (crp->crp_flags & CRYPTO_F_IMBUF) {
820 if (m_copyback((struct mbuf *)crp->crp_buf,
821 crde->crd_skip, crde->crd_len, buf, M_NOWAIT)) {
822 error = ENOMEM;
823 goto out;
824 }
825 } else {
826 cuio_copyback((struct uio *)crp->crp_buf,
827 crde->crd_skip, crde->crd_len, buf);
828 }
829
830 /* Copy back the authentication tag. */
831 if (crp->crp_flags & CRYPTO_F_IMBUF) {
832 if (m_copyback((struct mbuf *)crp->crp_buf, crda->crd_inject,
833 GMAC_DIGEST_LEN, tag, M_NOWAIT)) {
834 error = ENOMEM;
835 goto out;
836 }
837 } else {
838 memcpy(crp->crp_mac, tag, GMAC_DIGEST_LEN);
839 }
840
841 out:
842 explicit_bzero(buf, rlen);
843 explicit_bzero(icb, sizeof(icb));
844 explicit_bzero(tag, sizeof(tag));
845
846 return error;
847 }
848
849 void
octcrypto_hmac(struct cryptodesc * crda,uint8_t * buf,size_t len,struct octcrypto_session * ses,uint64_t * res)850 octcrypto_hmac(struct cryptodesc *crda, uint8_t *buf, size_t len,
851 struct octcrypto_session *ses, uint64_t *res)
852 {
853 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)];
854 uint8_t *bptr = (uint8_t *)block;
855 const struct octcrypto_hmac *hmac = ses->ses_hmac;
856 size_t left;
857
858 cop2_enable();
859
860 /*
861 * Compute the inner hash.
862 */
863
864 hmac->set_iv(ses->ses_iiv);
865 hmac->transform(buf, len);
866
867 memset(block, 0, hmac->blocklen);
868 left = len & (hmac->blocklen - 1);
869 bptr[left] = 0x80;
870 if (left > 0) {
871 memcpy(block, buf + len - left, left);
872
873 if (roundup(left + 1, hmac->countlen) >
874 (hmac->blocklen - hmac->countlen)) {
875 hmac->transform(block, hmac->blocklen);
876 memset(block, 0, hmac->blocklen);
877 }
878 }
879
880 switch (crda->crd_alg) {
881 case CRYPTO_MD5_HMAC:
882 block[7] = htole64((64 + len) * 8);
883 break;
884 case CRYPTO_SHA1_HMAC:
885 case CRYPTO_SHA2_256_HMAC:
886 block[7] = htobe64((64 + len) * 8);
887 break;
888 case CRYPTO_SHA2_384_HMAC:
889 case CRYPTO_SHA2_512_HMAC:
890 block[15] = htobe64((128 + len) * 8);
891 break;
892 }
893
894 hmac->transform(block, hmac->blocklen);
895
896 /*
897 * Compute the outer hash.
898 */
899
900 memset(block, 0, hmac->blocklen);
901 hmac->get_iv(block);
902 hmac->set_iv(ses->ses_oiv);
903
904 switch (crda->crd_alg) {
905 case CRYPTO_MD5_HMAC:
906 block[2] = htobe64(1ULL << 63);
907 block[7] = htole64((64 + 16) * 8);
908 break;
909 case CRYPTO_SHA1_HMAC:
910 block[2] |= htobe64(1ULL << 31);
911 block[7] = htobe64((64 + 20) * 8);
912 break;
913 case CRYPTO_SHA2_256_HMAC:
914 block[4] = htobe64(1ULL << 63);
915 block[7] = htobe64((64 + 32) * 8);
916 break;
917 case CRYPTO_SHA2_384_HMAC:
918 /*
919 * The computed digest is 512 bits long.
920 * It has to be truncated to 384 bits.
921 */
922 block[6] = htobe64(1ULL << 63);
923 block[7] = 0; /* truncation */
924 block[15] = htobe64((128 + 48) * 8);
925 break;
926 case CRYPTO_SHA2_512_HMAC:
927 block[8] = htobe64(1ULL << 63);
928 block[15] = htobe64((128 + 64) * 8);
929 break;
930 }
931
932 hmac->transform(block, hmac->blocklen);
933 hmac->get_iv(res);
934 hmac->clear();
935
936 cop2_disable();
937
938 explicit_bzero(block, sizeof(block));
939 }
940
941 int
octcrypto_authenc_hmac(struct cryptop * crp,struct cryptodesc * crde,struct cryptodesc * crda,struct octcrypto_session * ses)942 octcrypto_authenc_hmac(struct cryptop *crp, struct cryptodesc *crde,
943 struct cryptodesc *crda, struct octcrypto_session *ses)
944 {
945 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)];
946 uint64_t iv[ndwords(EALG_MAX_BLOCK_LEN)];
947 uint64_t tag[ndwords(AALG_MAX_RESULT_LEN)];
948 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()];
949 uint8_t *buf, *authbuf, *encbuf;
950 size_t authlen;
951 size_t buflen;
952 size_t len;
953 size_t skip;
954 off_t authskip = 0;
955 off_t encskip = 0;
956 int error = 0;
957 int ivlen;
958
959 if (crde != NULL && crda != NULL) {
960 skip = MIN(crde->crd_skip, crda->crd_skip);
961 len = MAX(crde->crd_skip + crde->crd_len,
962 crda->crd_skip + crda->crd_len) - skip;
963
964 if (crda->crd_skip < crde->crd_skip)
965 encskip = crde->crd_skip - crda->crd_skip;
966 else
967 authskip = crda->crd_skip - crde->crd_skip;
968 } else if (crde != NULL) {
969 skip = crde->crd_skip;
970 len = crde->crd_len;
971 } else {
972 KASSERT(crda != NULL);
973
974 skip = crda->crd_skip;
975 len = crda->crd_len;
976 }
977
978 buflen = len;
979
980 /* Reserve space for ESN. */
981 if (crda != NULL && (crda->crd_flags & CRD_F_ESN) != 0)
982 buflen += 4;
983
984 buflen = roundup(buflen, EALG_MAX_BLOCK_LEN);
985 if (buflen > pcpu->pcpu_buflen) {
986 if (pcpu->pcpu_buf != NULL) {
987 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen);
988 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen);
989 }
990 pcpu->pcpu_buflen = 0;
991 pcpu->pcpu_buf = malloc(buflen, M_DEVBUF, M_NOWAIT | M_ZERO);
992 if (pcpu->pcpu_buf == NULL)
993 return ENOMEM;
994 pcpu->pcpu_buflen = buflen;
995 }
996 buf = pcpu->pcpu_buf;
997
998 authbuf = buf + authskip;
999 encbuf = buf + encskip;
1000
1001 /* Prepare the IV. */
1002 if (crde != NULL) {
1003 /* CBC uses 16 bytes, CTR 8 bytes. */
1004 ivlen = (crde->crd_alg == CRYPTO_AES_CBC) ? 16 : 8;
1005
1006 if (crde->crd_flags & CRD_F_ENCRYPT) {
1007 if (crde->crd_flags & CRD_F_IV_EXPLICIT)
1008 memcpy(iv, crde->crd_iv, ivlen);
1009 else
1010 arc4random_buf(iv, ivlen);
1011
1012 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) {
1013 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1014 if (m_copyback(
1015 (struct mbuf *)crp->crp_buf,
1016 crde->crd_inject, ivlen, iv,
1017 M_NOWAIT)) {
1018 error = ENOMEM;
1019 goto out;
1020 }
1021 } else {
1022 cuio_copyback(
1023 (struct uio *)crp->crp_buf,
1024 crde->crd_inject, ivlen, iv);
1025 }
1026 }
1027 } else {
1028 if (crde->crd_flags & CRD_F_IV_EXPLICIT) {
1029 memcpy(iv, crde->crd_iv, ivlen);
1030 } else {
1031 if (crp->crp_flags & CRYPTO_F_IMBUF)
1032 m_copydata((struct mbuf *)crp->crp_buf,
1033 crde->crd_inject, ivlen, iv);
1034 else
1035 cuio_copydata(
1036 (struct uio *)crp->crp_buf,
1037 crde->crd_inject, ivlen,
1038 (uint8_t *)iv);
1039 }
1040 }
1041 }
1042
1043 /* Copy input to the working buffer. */
1044 if (crp->crp_flags & CRYPTO_F_IMBUF)
1045 m_copydata((struct mbuf *)crp->crp_buf, skip, len, buf);
1046 else
1047 cuio_copydata((struct uio *)crp->crp_buf, skip, len, buf);
1048
1049 /* If ESN is used, append it to the buffer. */
1050 if (crda != NULL) {
1051 authlen = crda->crd_len;
1052 if (crda->crd_flags & CRD_F_ESN) {
1053 memcpy(buf + len, crda->crd_esn, 4);
1054 authlen += 4;
1055 }
1056 }
1057
1058 if (crde != NULL) {
1059 /* Compute authentication tag before decryption. */
1060 if (crda != NULL && (crde->crd_flags & CRD_F_ENCRYPT) == 0)
1061 octcrypto_hmac(crda, authbuf, authlen, ses, tag);
1062
1063 /* Apply the cipher. */
1064 switch (crde->crd_alg) {
1065 case CRYPTO_AES_CBC:
1066 cop2_enable();
1067 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen);
1068 if (crde->crd_flags & CRD_F_ENCRYPT)
1069 octcrypto_aes_cbc_enc(encbuf, crde->crd_len,
1070 iv);
1071 else
1072 octcrypto_aes_cbc_dec(encbuf, crde->crd_len,
1073 iv);
1074 octcrypto_aes_clear();
1075 cop2_disable();
1076 break;
1077
1078 case CRYPTO_AES_CTR:
1079 memset(icb, 0, sizeof(icb));
1080 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE);
1081 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv,
1082 AESCTR_IVSIZE);
1083 cop2_enable();
1084 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen);
1085 octcrypto_aes_ctr_enc(encbuf, crde->crd_len, icb);
1086 octcrypto_aes_clear();
1087 cop2_disable();
1088 explicit_bzero(icb, sizeof(icb));
1089 break;
1090 }
1091
1092 /* Copy back the output. */
1093 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1094 if (m_copyback((struct mbuf *)crp->crp_buf,
1095 crde->crd_skip, crde->crd_len, encbuf, M_NOWAIT)) {
1096 error = ENOMEM;
1097 goto out;
1098 }
1099 } else {
1100 cuio_copyback((struct uio *)crp->crp_buf,
1101 crde->crd_skip, crde->crd_len, encbuf);
1102 }
1103 }
1104
1105 if (crda != NULL) {
1106 /*
1107 * Compute authentication tag after encryption.
1108 * This also handles the authentication only case.
1109 */
1110 if (crde == NULL || (crde->crd_flags & CRD_F_ENCRYPT) != 0)
1111 octcrypto_hmac(crda, authbuf, authlen, ses, tag);
1112
1113 /* Copy back the authentication tag. */
1114 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1115 if (m_copyback((struct mbuf *)crp->crp_buf,
1116 crda->crd_inject, ses->ses_hmac->taglen, tag,
1117 M_NOWAIT)) {
1118 error = ENOMEM;
1119 goto out;
1120 }
1121 } else {
1122 memcpy(crp->crp_mac, tag, ses->ses_hmac->taglen);
1123 }
1124
1125 explicit_bzero(tag, sizeof(tag));
1126 }
1127
1128 out:
1129 explicit_bzero(buf, len);
1130 return error;
1131 }
1132
1133 void
octcrypto_ghash_update_md(GHASH_CTX * ghash,uint8_t * src,size_t len)1134 octcrypto_ghash_update_md(GHASH_CTX *ghash, uint8_t *src, size_t len)
1135 {
1136 CTASSERT(offsetof(GHASH_CTX, H) % 8 == 0);
1137 CTASSERT(offsetof(GHASH_CTX, S) % 8 == 0);
1138
1139 cop2_enable();
1140 octcrypto_ghash_init((uint64_t *)ghash->H, (uint64_t *)ghash->S);
1141 octcrypto_ghash_update(src, len);
1142 octcrypto_ghash_finish((uint64_t *)ghash->S);
1143 cop2_disable();
1144 }
1145