1 /* $OpenBSD: octcrypto.c,v 1.6 2021/10/13 13:08:58 bluhm Exp $ */ 2 3 /* 4 * Copyright (c) 2018 Visa Hankala 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Driver for the OCTEON cryptographic unit. 21 */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/atomic.h> 26 #include <sys/malloc.h> 27 #include <sys/mbuf.h> 28 #include <sys/pool.h> 29 #include <sys/smr.h> 30 #include <sys/tree.h> 31 32 #include <crypto/cryptodev.h> 33 #include <crypto/cryptosoft.h> 34 #include <crypto/xform.h> 35 36 #include <mips64/mips_cpu.h> 37 38 #include <machine/octeonvar.h> 39 40 /* Maximum number of dwords in hash IV. */ 41 #define MAX_IVNW 8 42 43 /* Number of dwords needed to cover `n' bytes. */ 44 #define ndwords(n) (roundup(n, 8) / (8)) 45 46 struct octcrypto_softc; 47 48 struct octcrypto_hmac { 49 void (*transform)(const void *, size_t); 50 void (*get_iv)(uint64_t *); 51 void (*set_iv)(const uint64_t *); 52 void (*clear)(void); 53 uint16_t blocklen; 54 uint16_t taglen; 55 uint16_t countlen; 56 }; 57 58 struct octcrypto_session { 59 uint32_t ses_sid; /* RB key, keep first */ 60 RBT_ENTRY(octrcypto_session) 61 ses_entry; 62 struct octcrypto_softc *ses_sc; 63 struct smr_entry ses_smr; 64 65 /* AES parameters */ 66 uint64_t ses_key[4]; 67 int ses_klen; 68 uint8_t ses_nonce[AESCTR_NONCESIZE]; 69 70 /* HMAC parameters */ 71 const struct octcrypto_hmac 72 *ses_hmac; 73 uint64_t ses_iiv[MAX_IVNW]; /* HMAC inner IV */ 74 uint64_t ses_oiv[MAX_IVNW]; /* HMAC outer IV */ 75 76 /* GHASH parameters */ 77 uint64_t ses_ghkey[2]; 78 79 struct swcr_data *ses_swd; 80 }; 81 82 struct octcrypto_cpu { 83 uint8_t *pcpu_buf; 84 size_t pcpu_buflen; 85 }; 86 87 struct octcrypto_softc { 88 struct device sc_dev; 89 int32_t sc_cid; 90 uint32_t sc_sid; 91 struct mutex sc_mtx; 92 RBT_HEAD(octcrypto_tree, octcrypto_session) 93 sc_sessions; 94 struct octcrypto_cpu sc_cpu[MAXCPUS]; 95 }; 96 97 int octcrypto_match(struct device *, void *, void *); 98 void octcrypto_attach(struct device *, struct device *, void *); 99 100 int octcrypto_newsession(uint32_t *, struct cryptoini *); 101 int octcrypto_freesession(uint64_t); 102 int octcrypto_process(struct cryptop *); 103 104 struct octcrypto_session * 105 octcrypto_get(struct octcrypto_softc *, uint32_t); 106 void octcrypto_free(struct octcrypto_session *); 107 void octcrypto_free_smr(void *); 108 109 void octcrypto_hmac(struct cryptodesc *, uint8_t *, size_t, 110 struct octcrypto_session *, uint64_t *); 111 int octcrypto_authenc_gmac(struct cryptop *, struct cryptodesc *, 112 struct cryptodesc *, struct octcrypto_session *); 113 int octcrypto_authenc_hmac(struct cryptop *, struct cryptodesc *, 114 struct cryptodesc *, struct octcrypto_session *); 115 int octcrypto_swauth(struct cryptop *, struct cryptodesc *, 116 struct swcr_data *, uint8_t *); 117 118 void octcrypto_ghash_update_md(GHASH_CTX *, uint8_t *, size_t); 119 120 void octcrypto_aes_clear(void); 121 void octcrypto_aes_cbc_dec(void *, size_t, const void *); 122 void octcrypto_aes_cbc_enc(void *, size_t, const void *); 123 void octcrypto_aes_ctr_enc(void *, size_t, const void *); 124 void octcrypto_aes_enc(uint64_t *); 125 void octcrypto_aes_set_key(const uint64_t *, int); 126 127 void octcrypto_ghash_finish(uint64_t *); 128 void octcrypto_ghash_init(const uint64_t *, const uint64_t *); 129 void octcrypto_ghash_update(const void *, size_t); 130 131 void octcrypto_hash_md5(const void *, size_t); 132 void octcrypto_hash_sha1(const void *, size_t); 133 void octcrypto_hash_sha256(const void *, size_t); 134 void octcrypto_hash_sha512(const void *, size_t); 135 void octcrypto_hash_clearn(void); 136 void octcrypto_hash_clearw(void); 137 void octcrypto_hash_get_ivn(uint64_t *); 138 void octcrypto_hash_get_ivw(uint64_t *); 139 void octcrypto_hash_set_ivn(const uint64_t *); 140 void octcrypto_hash_set_ivw(const uint64_t *); 141 142 const struct cfattach octcrypto_ca = { 143 sizeof(struct octcrypto_softc), octcrypto_match, octcrypto_attach 144 }; 145 146 struct cfdriver octcrypto_cd = { 147 NULL, "octcrypto", DV_DULL 148 }; 149 150 static const struct octcrypto_hmac hmac_md5_96 = { 151 .transform = octcrypto_hash_md5, 152 .get_iv = octcrypto_hash_get_ivn, 153 .set_iv = octcrypto_hash_set_ivn, 154 .clear = octcrypto_hash_clearn, 155 .blocklen = 64, 156 .taglen = 12, 157 .countlen = 8 158 }; 159 160 static const struct octcrypto_hmac hmac_sha1_96 = { 161 .transform = octcrypto_hash_sha1, 162 .get_iv = octcrypto_hash_get_ivn, 163 .set_iv = octcrypto_hash_set_ivn, 164 .clear = octcrypto_hash_clearn, 165 .blocklen = 64, 166 .taglen = 12, 167 .countlen = 8 168 }; 169 170 static const struct octcrypto_hmac hmac_sha2_256_128 = { 171 .transform = octcrypto_hash_sha256, 172 .get_iv = octcrypto_hash_get_ivn, 173 .set_iv = octcrypto_hash_set_ivn, 174 .clear = octcrypto_hash_clearn, 175 .blocklen = 64, 176 .taglen = 16, 177 .countlen = 8 178 }; 179 180 static const struct octcrypto_hmac hmac_sha2_384_192 = { 181 .transform = octcrypto_hash_sha512, 182 .get_iv = octcrypto_hash_get_ivw, 183 .set_iv = octcrypto_hash_set_ivw, 184 .clear = octcrypto_hash_clearw, 185 .blocklen = 128, 186 .taglen = 24, 187 .countlen = 16 188 }; 189 190 static const struct octcrypto_hmac hmac_sha2_512_256 = { 191 .transform = octcrypto_hash_sha512, 192 .get_iv = octcrypto_hash_get_ivw, 193 .set_iv = octcrypto_hash_set_ivw, 194 .clear = octcrypto_hash_clearw, 195 .blocklen = 128, 196 .taglen = 32, 197 .countlen = 16 198 }; 199 200 static struct pool octcryptopl; 201 static struct octcrypto_softc *octcrypto_sc; 202 203 static inline int 204 octcrypto_cmp(const struct octcrypto_session *a, 205 const struct octcrypto_session *b) 206 { 207 if (a->ses_sid < b->ses_sid) 208 return -1; 209 if (a->ses_sid > b->ses_sid) 210 return 1; 211 return 0; 212 } 213 214 RBT_PROTOTYPE(octcrypto_tree, octcrypto_session, sess_entry, octcrypto_cmp); 215 RBT_GENERATE(octcrypto_tree, octcrypto_session, ses_entry, octcrypto_cmp); 216 217 static inline void 218 cop2_enable(void) 219 { 220 setsr(getsr() | SR_COP_2_BIT); 221 } 222 223 static inline void 224 cop2_disable(void) 225 { 226 setsr(getsr() & ~SR_COP_2_BIT); 227 } 228 229 int 230 octcrypto_match(struct device *parent, void *match, void *aux) 231 { 232 return 1; 233 } 234 235 void 236 octcrypto_attach(struct device *parent, struct device *self, void *aux) 237 { 238 int algs[CRYPTO_ALGORITHM_MAX + 1]; 239 struct octcrypto_softc *sc = (struct octcrypto_softc *)self; 240 241 pool_init(&octcryptopl, sizeof(struct octcrypto_session), 0, IPL_VM, 0, 242 "octcrypto", NULL); 243 pool_setlowat(&octcryptopl, 2); 244 245 mtx_init(&sc->sc_mtx, IPL_VM); 246 RBT_INIT(octcrypto_tree, &sc->sc_sessions); 247 248 sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_MPSAFE); 249 if (sc->sc_cid < 0) { 250 printf(": could not get driver id\n"); 251 return; 252 } 253 254 printf("\n"); 255 256 memset(algs, 0, sizeof(algs)); 257 258 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED; 259 algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED; 260 algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED; 261 262 algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 263 algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 264 algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 265 algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 266 267 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 268 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 269 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 270 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 271 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 272 273 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 274 275 algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED; 276 277 octcrypto_sc = sc; 278 279 crypto_register(sc->sc_cid, algs, octcrypto_newsession, 280 octcrypto_freesession, octcrypto_process); 281 282 ghash_update = octcrypto_ghash_update_md; 283 } 284 285 struct octcrypto_session * 286 octcrypto_get(struct octcrypto_softc *sc, uint32_t sid) 287 { 288 struct octcrypto_session *ses; 289 290 SMR_ASSERT_CRITICAL(); 291 292 mtx_enter(&sc->sc_mtx); 293 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 294 (struct octcrypto_session *)&sid); 295 mtx_leave(&sc->sc_mtx); 296 return ses; 297 } 298 299 void 300 octcrypto_free(struct octcrypto_session *ses) 301 { 302 const struct auth_hash *axf; 303 struct swcr_data *swd; 304 305 if (ses->ses_swd != NULL) { 306 swd = ses->ses_swd; 307 axf = swd->sw_axf; 308 309 if (swd->sw_ictx != NULL) { 310 explicit_bzero(swd->sw_ictx, axf->ctxsize); 311 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize); 312 } 313 if (swd->sw_octx != NULL) { 314 explicit_bzero(swd->sw_octx, axf->ctxsize); 315 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize); 316 } 317 free(swd, M_CRYPTO_DATA, sizeof(*swd)); 318 } 319 320 explicit_bzero(ses, sizeof(*ses)); 321 pool_put(&octcryptopl, ses); 322 } 323 324 void 325 octcrypto_free_smr(void *arg) 326 { 327 struct octcrypto_session *ses = arg; 328 329 octcrypto_free(ses); 330 } 331 332 int 333 octcrypto_newsession(uint32_t *sidp, struct cryptoini *cri) 334 { 335 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 336 const struct auth_hash *axf; 337 struct cryptoini *c; 338 const struct octcrypto_hmac *hmac = NULL; 339 struct octcrypto_softc *sc = octcrypto_sc; 340 struct octcrypto_session *ses = NULL; 341 struct swcr_data *swd; 342 uint8_t *bptr; 343 size_t klen; 344 int i; 345 uint32_t sid; 346 347 if (sidp == NULL || cri == NULL) 348 return EINVAL; 349 350 ses = pool_get(&octcryptopl, PR_NOWAIT | PR_ZERO); 351 if (ses == NULL) 352 return ENOMEM; 353 ses->ses_sc = sc; 354 smr_init(&ses->ses_smr); 355 356 for (c = cri; c != NULL; c = c->cri_next) { 357 switch (c->cri_alg) { 358 case CRYPTO_AES_CBC: 359 ses->ses_klen = c->cri_klen / 8; 360 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 361 break; 362 363 case CRYPTO_AES_CTR: 364 case CRYPTO_AES_GCM_16: 365 case CRYPTO_AES_GMAC: 366 ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE; 367 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 368 memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen, 369 AESCTR_NONCESIZE); 370 break; 371 372 case CRYPTO_AES_128_GMAC: 373 case CRYPTO_AES_192_GMAC: 374 case CRYPTO_AES_256_GMAC: 375 cop2_enable(); 376 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 377 octcrypto_aes_enc(ses->ses_ghkey); 378 octcrypto_aes_clear(); 379 cop2_disable(); 380 break; 381 382 case CRYPTO_MD5_HMAC: 383 ses->ses_iiv[0] = 0x0123456789abcdefULL; 384 ses->ses_iiv[1] = 0xfedcba9876543210ULL; 385 ses->ses_hmac = &hmac_md5_96; 386 goto hwauthcommon; 387 388 case CRYPTO_SHA1_HMAC: 389 ses->ses_iiv[0] = 0x67452301efcdab89ULL; 390 ses->ses_iiv[1] = 0x98badcfe10325476ULL; 391 ses->ses_iiv[2] = 0xc3d2e1f000000000ULL; 392 ses->ses_hmac = &hmac_sha1_96; 393 goto hwauthcommon; 394 395 case CRYPTO_SHA2_256_HMAC: 396 ses->ses_iiv[0] = 0x6a09e667bb67ae85ULL; 397 ses->ses_iiv[1] = 0x3c6ef372a54ff53aULL; 398 ses->ses_iiv[2] = 0x510e527f9b05688cULL; 399 ses->ses_iiv[3] = 0x1f83d9ab5be0cd19ULL; 400 ses->ses_hmac = &hmac_sha2_256_128; 401 goto hwauthcommon; 402 403 case CRYPTO_SHA2_384_HMAC: 404 ses->ses_iiv[0] = 0xcbbb9d5dc1059ed8ULL; 405 ses->ses_iiv[1] = 0x629a292a367cd507ULL; 406 ses->ses_iiv[2] = 0x9159015a3070dd17ULL; 407 ses->ses_iiv[3] = 0x152fecd8f70e5939ULL; 408 ses->ses_iiv[4] = 0x67332667ffc00b31ULL; 409 ses->ses_iiv[5] = 0x8eb44a8768581511ULL; 410 ses->ses_iiv[6] = 0xdb0c2e0d64f98fa7ULL; 411 ses->ses_iiv[7] = 0x47b5481dbefa4fa4ULL; 412 ses->ses_hmac = &hmac_sha2_384_192; 413 goto hwauthcommon; 414 415 case CRYPTO_SHA2_512_HMAC: 416 ses->ses_iiv[0] = 0x6a09e667f3bcc908ULL; 417 ses->ses_iiv[1] = 0xbb67ae8584caa73bULL; 418 ses->ses_iiv[2] = 0x3c6ef372fe94f82bULL; 419 ses->ses_iiv[3] = 0xa54ff53a5f1d36f1ULL; 420 ses->ses_iiv[4] = 0x510e527fade682d1ULL; 421 ses->ses_iiv[5] = 0x9b05688c2b3e6c1fULL; 422 ses->ses_iiv[6] = 0x1f83d9abfb41bd6bULL; 423 ses->ses_iiv[7] = 0x5be0cd19137e2179ULL; 424 ses->ses_hmac = &hmac_sha2_512_256; 425 426 hwauthcommon: 427 memcpy(ses->ses_oiv, ses->ses_iiv, 428 sizeof(uint64_t) * MAX_IVNW); 429 430 bptr = (char *)block; 431 klen = c->cri_klen / 8; 432 hmac = ses->ses_hmac; 433 434 memcpy(bptr, c->cri_key, klen); 435 memset(bptr + klen, 0, hmac->blocklen - klen); 436 for (i = 0; i < hmac->blocklen; i++) 437 bptr[i] ^= HMAC_IPAD_VAL; 438 439 cop2_enable(); 440 hmac->set_iv(ses->ses_iiv); 441 hmac->transform(block, hmac->blocklen); 442 hmac->get_iv(ses->ses_iiv); 443 444 for (i = 0; i < hmac->blocklen; i++) 445 bptr[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 446 447 hmac->set_iv(ses->ses_oiv); 448 hmac->transform(block, hmac->blocklen); 449 hmac->get_iv(ses->ses_oiv); 450 hmac->clear(); 451 cop2_disable(); 452 453 explicit_bzero(block, hmac->blocklen); 454 break; 455 456 case CRYPTO_RIPEMD160_HMAC: 457 axf = &auth_hash_hmac_ripemd_160_96; 458 goto swauthcommon; 459 460 swauthcommon: 461 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 462 M_NOWAIT | M_ZERO); 463 if (swd == NULL) { 464 octcrypto_free(ses); 465 return ENOMEM; 466 } 467 ses->ses_swd = swd; 468 469 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 470 M_NOWAIT); 471 if (swd->sw_ictx == NULL) { 472 octcrypto_free(ses); 473 return ENOMEM; 474 } 475 476 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 477 M_NOWAIT); 478 if (swd->sw_octx == NULL) { 479 octcrypto_free(ses); 480 return ENOMEM; 481 } 482 483 for (i = 0; i < c->cri_klen / 8; i++) 484 c->cri_key[i] ^= HMAC_IPAD_VAL; 485 486 axf->Init(swd->sw_ictx); 487 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 488 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 489 axf->blocksize - (c->cri_klen / 8)); 490 491 for (i = 0; i < c->cri_klen / 8; i++) 492 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 493 HMAC_OPAD_VAL); 494 495 axf->Init(swd->sw_octx); 496 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 497 axf->Update(swd->sw_octx, hmac_opad_buffer, 498 axf->blocksize - (c->cri_klen / 8)); 499 500 for (i = 0; i < c->cri_klen / 8; i++) 501 c->cri_key[i] ^= HMAC_OPAD_VAL; 502 503 swd->sw_axf = axf; 504 swd->sw_alg = c->cri_alg; 505 506 break; 507 508 case CRYPTO_ESN: 509 /* nothing to do */ 510 break; 511 512 default: 513 octcrypto_free(ses); 514 return EINVAL; 515 } 516 } 517 518 mtx_enter(&sc->sc_mtx); 519 /* Find a free session ID. Assume there is one. */ 520 do { 521 sc->sc_sid++; 522 if (sc->sc_sid == 0) 523 sc->sc_sid = 1; 524 sid = sc->sc_sid; 525 } while (RBT_FIND(octcrypto_tree, &sc->sc_sessions, 526 (struct octcrypto_session *)&sid) != NULL); 527 ses->ses_sid = sid; 528 RBT_INSERT(octcrypto_tree, &sc->sc_sessions, ses); 529 mtx_leave(&sc->sc_mtx); 530 531 *sidp = ses->ses_sid; 532 return 0; 533 } 534 535 int 536 octcrypto_freesession(uint64_t tid) 537 { 538 struct octcrypto_softc *sc = octcrypto_sc; 539 struct octcrypto_session *ses; 540 uint32_t sid = (uint32_t)tid; 541 542 mtx_enter(&sc->sc_mtx); 543 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 544 (struct octcrypto_session *)&sid); 545 if (ses != NULL) 546 RBT_REMOVE(octcrypto_tree, &sc->sc_sessions, ses); 547 mtx_leave(&sc->sc_mtx); 548 549 if (ses == NULL) 550 return EINVAL; 551 552 smr_call(&ses->ses_smr, octcrypto_free_smr, ses); 553 554 return 0; 555 } 556 557 enum { 558 ALG_UNHANDLED, 559 ALG_AES, 560 ALG_AES_GHASH, 561 ALG_GMAC, 562 ALG_HMAC 563 }; 564 565 static int 566 alg_class(int alg) 567 { 568 switch (alg) { 569 case CRYPTO_AES_CBC: 570 case CRYPTO_AES_CTR: 571 return ALG_AES; 572 case CRYPTO_AES_GCM_16: 573 case CRYPTO_AES_GMAC: 574 return ALG_AES_GHASH; 575 case CRYPTO_AES_128_GMAC: 576 case CRYPTO_AES_192_GMAC: 577 case CRYPTO_AES_256_GMAC: 578 return ALG_GMAC; 579 case CRYPTO_MD5_HMAC: 580 case CRYPTO_SHA1_HMAC: 581 case CRYPTO_SHA2_256_HMAC: 582 case CRYPTO_SHA2_384_HMAC: 583 case CRYPTO_SHA2_512_HMAC: 584 return ALG_HMAC; 585 default: 586 return ALG_UNHANDLED; 587 } 588 } 589 590 int 591 octcrypto_process(struct cryptop *crp) 592 { 593 struct cryptodesc *crd, *crd2; 594 struct octcrypto_softc *sc = octcrypto_sc; 595 struct octcrypto_session *ses = NULL; 596 int alg, alg2; 597 int error = 0; 598 int i; 599 600 KASSERT(crp->crp_ndesc >= 1); 601 602 smr_read_enter(); 603 ses = octcrypto_get(sc, (uint32_t)crp->crp_sid); 604 if (ses == NULL) { 605 error = EINVAL; 606 goto out; 607 } 608 609 if (crp->crp_ndesc == 2) { 610 crd = &crp->crp_desc[0]; 611 crd2 = &crp->crp_desc[1]; 612 alg = alg_class(crd->crd_alg); 613 alg2 = alg_class(crd2->crd_alg); 614 615 if ((alg == ALG_AES) && (alg2 == ALG_HMAC)) { 616 error = octcrypto_authenc_hmac(crp, crd, crd2, ses); 617 goto out; 618 } else if ((alg2 == ALG_AES) && (alg == ALG_HMAC)) { 619 error = octcrypto_authenc_hmac(crp, crd2, crd, ses); 620 goto out; 621 } else if ((alg == ALG_AES_GHASH) && (alg2 == ALG_GMAC)) { 622 error = octcrypto_authenc_gmac(crp, crd, crd2, ses); 623 goto out; 624 } else if ((alg2 == ALG_AES_GHASH) && (alg == ALG_GMAC)) { 625 error = octcrypto_authenc_gmac(crp, crd2, crd, ses); 626 goto out; 627 } 628 } 629 630 for (i = 0; i < crp->crp_ndesc; i++) { 631 crd = &crp->crp_desc[i]; 632 switch (crd->crd_alg) { 633 case CRYPTO_AES_CBC: 634 case CRYPTO_AES_CTR: 635 error = octcrypto_authenc_hmac(crp, crd, NULL, ses); 636 break; 637 638 case CRYPTO_MD5_HMAC: 639 case CRYPTO_SHA1_HMAC: 640 case CRYPTO_SHA2_256_HMAC: 641 case CRYPTO_SHA2_384_HMAC: 642 case CRYPTO_SHA2_512_HMAC: 643 error = octcrypto_authenc_hmac(crp, NULL, crd, ses); 644 break; 645 646 case CRYPTO_RIPEMD160_HMAC: 647 error = octcrypto_swauth(crp, crd, ses->ses_swd, 648 crp->crp_buf); 649 break; 650 651 default: 652 error = EINVAL; 653 break; 654 } 655 } 656 657 out: 658 smr_read_leave(); 659 660 crp->crp_etype = error; 661 crypto_done(crp); 662 return error; 663 } 664 665 int 666 octcrypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 667 struct swcr_data *sw, uint8_t *buf) 668 { 669 int type; 670 671 if (crp->crp_flags & CRYPTO_F_IMBUF) 672 type = CRYPTO_BUF_MBUF; 673 else 674 type = CRYPTO_BUF_IOV; 675 676 return swcr_authcompute(crp, crd, sw, buf, type); 677 } 678 679 int 680 octcrypto_authenc_gmac(struct cryptop *crp, struct cryptodesc *crde, 681 struct cryptodesc *crda, struct octcrypto_session *ses) 682 { 683 uint64_t block[ndwords(AESCTR_BLOCKSIZE)]; 684 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 685 uint64_t iv[ndwords(AESCTR_BLOCKSIZE)]; 686 uint64_t tag[ndwords(GMAC_BLOCK_LEN)]; 687 uint8_t *buf; 688 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 689 size_t aadlen; 690 size_t ivlen = 8; 691 size_t rlen; 692 int error = 0; 693 unsigned int iskip = 0; 694 unsigned int oskip = 0; 695 696 KASSERT(crda != NULL); 697 KASSERT(crde != NULL); 698 699 rlen = roundup(crde->crd_len, AESCTR_BLOCKSIZE); 700 if (rlen > pcpu->pcpu_buflen) { 701 if (pcpu->pcpu_buf != NULL) { 702 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 703 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 704 } 705 pcpu->pcpu_buflen = 0; 706 pcpu->pcpu_buf = malloc(rlen, M_DEVBUF, M_NOWAIT | M_ZERO); 707 if (pcpu->pcpu_buf == NULL) 708 return ENOMEM; 709 pcpu->pcpu_buflen = rlen; 710 } 711 buf = pcpu->pcpu_buf; 712 713 /* Prepare the IV. */ 714 if (crde->crd_flags & CRD_F_ENCRYPT) { 715 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 716 memcpy(iv, crde->crd_iv, ivlen); 717 else 718 arc4random_buf(iv, ivlen); 719 720 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 721 if (crp->crp_flags & CRYPTO_F_IMBUF) { 722 if (m_copyback((struct mbuf *)crp->crp_buf, 723 crde->crd_inject, ivlen, (uint8_t *)iv, 724 M_NOWAIT)) { 725 error = ENOMEM; 726 goto out; 727 } 728 } else { 729 cuio_copyback((struct uio *)crp->crp_buf, 730 crde->crd_inject, ivlen, (uint8_t *)iv); 731 } 732 } 733 } else { 734 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 735 memcpy(iv, crde->crd_iv, ivlen); 736 } else { 737 if (crp->crp_flags & CRYPTO_F_IMBUF) 738 m_copydata((struct mbuf *)crp->crp_buf, 739 crde->crd_inject, ivlen, iv); 740 else 741 cuio_copydata((struct uio *)crp->crp_buf, 742 crde->crd_inject, ivlen, (uint8_t *)iv); 743 } 744 } 745 746 memset(icb, 0, sizeof(icb)); 747 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 748 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE); 749 ((uint8_t *)icb)[AESCTR_BLOCKSIZE - 1] = 1; 750 751 /* Prepare the AAD. */ 752 aadlen = crda->crd_len; 753 if (crda->crd_flags & CRD_F_ESN) { 754 aadlen += 4; 755 if (crp->crp_flags & CRYPTO_F_IMBUF) 756 m_copydata((struct mbuf *)crp->crp_buf, 757 crda->crd_skip, 4, buf); 758 else 759 cuio_copydata((struct uio *)crp->crp_buf, 760 crda->crd_skip, 4, buf); 761 memcpy(buf + 4, crda->crd_esn, 4); 762 iskip = 4; 763 oskip = 8; 764 } 765 if (crp->crp_flags & CRYPTO_F_IMBUF) 766 m_copydata((struct mbuf *)crp->crp_buf, 767 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 768 else 769 cuio_copydata((struct uio *)crp->crp_buf, 770 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 771 772 cop2_enable(); 773 octcrypto_ghash_init(ses->ses_ghkey, NULL); 774 octcrypto_ghash_update(buf, roundup(aadlen, GMAC_BLOCK_LEN)); 775 cop2_disable(); 776 777 memset(buf, 0, aadlen); 778 779 /* Copy input to the working buffer. */ 780 if (crp->crp_flags & CRYPTO_F_IMBUF) 781 m_copydata((struct mbuf *)crp->crp_buf, crde->crd_skip, 782 crde->crd_len, buf); 783 else 784 cuio_copydata((struct uio *)crp->crp_buf, crde->crd_skip, 785 crde->crd_len, buf); 786 787 cop2_enable(); 788 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 789 790 switch (crde->crd_alg) { 791 case CRYPTO_AES_GCM_16: 792 if (crde->crd_flags & CRD_F_ENCRYPT) { 793 octcrypto_aes_ctr_enc(buf, rlen, icb); 794 memset(buf + crde->crd_len, 0, rlen - crde->crd_len); 795 octcrypto_ghash_update(buf, rlen); 796 } else { 797 octcrypto_ghash_update(buf, rlen); 798 octcrypto_aes_ctr_enc(buf, rlen, icb); 799 } 800 break; 801 802 case CRYPTO_AES_GMAC: 803 octcrypto_ghash_update(buf, rlen); 804 break; 805 } 806 807 block[0] = htobe64(aadlen * 8); 808 block[1] = htobe64(crde->crd_len * 8); 809 octcrypto_ghash_update(block, GMAC_BLOCK_LEN); 810 octcrypto_ghash_finish(tag); 811 812 block[0] = icb[0]; 813 block[1] = icb[1]; 814 octcrypto_aes_enc(block); 815 tag[0] ^= block[0]; 816 tag[1] ^= block[1]; 817 818 octcrypto_aes_clear(); 819 cop2_disable(); 820 821 /* Copy back the output. */ 822 if (crp->crp_flags & CRYPTO_F_IMBUF) { 823 if (m_copyback((struct mbuf *)crp->crp_buf, 824 crde->crd_skip, crde->crd_len, buf, M_NOWAIT)) { 825 error = ENOMEM; 826 goto out; 827 } 828 } else { 829 cuio_copyback((struct uio *)crp->crp_buf, 830 crde->crd_skip, crde->crd_len, buf); 831 } 832 833 /* Copy back the authentication tag. */ 834 if (crp->crp_flags & CRYPTO_F_IMBUF) { 835 if (m_copyback((struct mbuf *)crp->crp_buf, crda->crd_inject, 836 GMAC_DIGEST_LEN, tag, M_NOWAIT)) { 837 error = ENOMEM; 838 goto out; 839 } 840 } else { 841 memcpy(crp->crp_mac, tag, GMAC_DIGEST_LEN); 842 } 843 844 out: 845 explicit_bzero(buf, rlen); 846 explicit_bzero(icb, sizeof(icb)); 847 explicit_bzero(tag, sizeof(tag)); 848 849 return error; 850 } 851 852 void 853 octcrypto_hmac(struct cryptodesc *crda, uint8_t *buf, size_t len, 854 struct octcrypto_session *ses, uint64_t *res) 855 { 856 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 857 uint8_t *bptr = (uint8_t *)block; 858 const struct octcrypto_hmac *hmac = ses->ses_hmac; 859 size_t left; 860 861 cop2_enable(); 862 863 /* 864 * Compute the inner hash. 865 */ 866 867 hmac->set_iv(ses->ses_iiv); 868 hmac->transform(buf, len); 869 870 memset(block, 0, hmac->blocklen); 871 left = len & (hmac->blocklen - 1); 872 bptr[left] = 0x80; 873 if (left > 0) { 874 memcpy(block, buf + len - left, left); 875 876 if (roundup(left + 1, hmac->countlen) > 877 (hmac->blocklen - hmac->countlen)) { 878 hmac->transform(block, hmac->blocklen); 879 memset(block, 0, hmac->blocklen); 880 } 881 } 882 883 switch (crda->crd_alg) { 884 case CRYPTO_MD5_HMAC: 885 block[7] = htole64((64 + len) * 8); 886 break; 887 case CRYPTO_SHA1_HMAC: 888 case CRYPTO_SHA2_256_HMAC: 889 block[7] = htobe64((64 + len) * 8); 890 break; 891 case CRYPTO_SHA2_384_HMAC: 892 case CRYPTO_SHA2_512_HMAC: 893 block[15] = htobe64((128 + len) * 8); 894 break; 895 } 896 897 hmac->transform(block, hmac->blocklen); 898 899 /* 900 * Compute the outer hash. 901 */ 902 903 memset(block, 0, hmac->blocklen); 904 hmac->get_iv(block); 905 hmac->set_iv(ses->ses_oiv); 906 907 switch (crda->crd_alg) { 908 case CRYPTO_MD5_HMAC: 909 block[2] = htobe64(1ULL << 63); 910 block[7] = htole64((64 + 16) * 8); 911 break; 912 case CRYPTO_SHA1_HMAC: 913 block[2] |= htobe64(1ULL << 31); 914 block[7] = htobe64((64 + 20) * 8); 915 break; 916 case CRYPTO_SHA2_256_HMAC: 917 block[4] = htobe64(1ULL << 63); 918 block[7] = htobe64((64 + 32) * 8); 919 break; 920 case CRYPTO_SHA2_384_HMAC: 921 /* 922 * The computed digest is 512 bits long. 923 * It has to be truncated to 384 bits. 924 */ 925 block[6] = htobe64(1ULL << 63); 926 block[7] = 0; /* truncation */ 927 block[15] = htobe64((128 + 48) * 8); 928 break; 929 case CRYPTO_SHA2_512_HMAC: 930 block[8] = htobe64(1ULL << 63); 931 block[15] = htobe64((128 + 64) * 8); 932 break; 933 } 934 935 hmac->transform(block, hmac->blocklen); 936 hmac->get_iv(res); 937 hmac->clear(); 938 939 cop2_disable(); 940 941 explicit_bzero(block, sizeof(block)); 942 } 943 944 int 945 octcrypto_authenc_hmac(struct cryptop *crp, struct cryptodesc *crde, 946 struct cryptodesc *crda, struct octcrypto_session *ses) 947 { 948 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 949 uint64_t iv[ndwords(EALG_MAX_BLOCK_LEN)]; 950 uint64_t tag[ndwords(AALG_MAX_RESULT_LEN)]; 951 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 952 uint8_t *buf, *authbuf, *encbuf; 953 size_t authlen; 954 size_t buflen; 955 size_t len; 956 size_t skip; 957 off_t authskip = 0; 958 off_t encskip = 0; 959 int error = 0; 960 int ivlen; 961 962 if (crde != NULL && crda != NULL) { 963 skip = MIN(crde->crd_skip, crda->crd_skip); 964 len = MAX(crde->crd_skip + crde->crd_len, 965 crda->crd_skip + crda->crd_len) - skip; 966 967 if (crda->crd_skip < crde->crd_skip) 968 encskip = crde->crd_skip - crda->crd_skip; 969 else 970 authskip = crda->crd_skip - crde->crd_skip; 971 } else if (crde != NULL) { 972 skip = crde->crd_skip; 973 len = crde->crd_len; 974 } else { 975 KASSERT(crda != NULL); 976 977 skip = crda->crd_skip; 978 len = crda->crd_len; 979 } 980 981 buflen = len; 982 983 /* Reserve space for ESN. */ 984 if (crda != NULL && (crda->crd_flags & CRD_F_ESN) != 0) 985 buflen += 4; 986 987 buflen = roundup(buflen, EALG_MAX_BLOCK_LEN); 988 if (buflen > pcpu->pcpu_buflen) { 989 if (pcpu->pcpu_buf != NULL) { 990 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 991 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 992 } 993 pcpu->pcpu_buflen = 0; 994 pcpu->pcpu_buf = malloc(buflen, M_DEVBUF, M_NOWAIT | M_ZERO); 995 if (pcpu->pcpu_buf == NULL) 996 return ENOMEM; 997 pcpu->pcpu_buflen = buflen; 998 } 999 buf = pcpu->pcpu_buf; 1000 1001 authbuf = buf + authskip; 1002 encbuf = buf + encskip; 1003 1004 /* Prepare the IV. */ 1005 if (crde != NULL) { 1006 /* CBC uses 16 bytes, CTR 8 bytes. */ 1007 ivlen = (crde->crd_alg == CRYPTO_AES_CBC) ? 16 : 8; 1008 1009 if (crde->crd_flags & CRD_F_ENCRYPT) { 1010 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 1011 memcpy(iv, crde->crd_iv, ivlen); 1012 else 1013 arc4random_buf(iv, ivlen); 1014 1015 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 1016 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1017 if (m_copyback( 1018 (struct mbuf *)crp->crp_buf, 1019 crde->crd_inject, ivlen, iv, 1020 M_NOWAIT)) { 1021 error = ENOMEM; 1022 goto out; 1023 } 1024 } else { 1025 cuio_copyback( 1026 (struct uio *)crp->crp_buf, 1027 crde->crd_inject, ivlen, iv); 1028 } 1029 } 1030 } else { 1031 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 1032 memcpy(iv, crde->crd_iv, ivlen); 1033 } else { 1034 if (crp->crp_flags & CRYPTO_F_IMBUF) 1035 m_copydata((struct mbuf *)crp->crp_buf, 1036 crde->crd_inject, ivlen, iv); 1037 else 1038 cuio_copydata( 1039 (struct uio *)crp->crp_buf, 1040 crde->crd_inject, ivlen, 1041 (uint8_t *)iv); 1042 } 1043 } 1044 } 1045 1046 /* Copy input to the working buffer. */ 1047 if (crp->crp_flags & CRYPTO_F_IMBUF) 1048 m_copydata((struct mbuf *)crp->crp_buf, skip, len, buf); 1049 else 1050 cuio_copydata((struct uio *)crp->crp_buf, skip, len, buf); 1051 1052 /* If ESN is used, append it to the buffer. */ 1053 if (crda != NULL) { 1054 authlen = crda->crd_len; 1055 if (crda->crd_flags & CRD_F_ESN) { 1056 memcpy(buf + len, crda->crd_esn, 4); 1057 authlen += 4; 1058 } 1059 } 1060 1061 if (crde != NULL) { 1062 /* Compute authentication tag before decryption. */ 1063 if (crda != NULL && (crde->crd_flags & CRD_F_ENCRYPT) == 0) 1064 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1065 1066 /* Apply the cipher. */ 1067 switch (crde->crd_alg) { 1068 case CRYPTO_AES_CBC: 1069 cop2_enable(); 1070 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1071 if (crde->crd_flags & CRD_F_ENCRYPT) 1072 octcrypto_aes_cbc_enc(encbuf, crde->crd_len, 1073 iv); 1074 else 1075 octcrypto_aes_cbc_dec(encbuf, crde->crd_len, 1076 iv); 1077 octcrypto_aes_clear(); 1078 cop2_disable(); 1079 break; 1080 1081 case CRYPTO_AES_CTR: 1082 memset(icb, 0, sizeof(icb)); 1083 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 1084 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, 1085 AESCTR_IVSIZE); 1086 cop2_enable(); 1087 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1088 octcrypto_aes_ctr_enc(encbuf, crde->crd_len, icb); 1089 octcrypto_aes_clear(); 1090 cop2_disable(); 1091 explicit_bzero(icb, sizeof(icb)); 1092 break; 1093 } 1094 1095 /* Copy back the output. */ 1096 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1097 if (m_copyback((struct mbuf *)crp->crp_buf, 1098 crde->crd_skip, crde->crd_len, encbuf, M_NOWAIT)) { 1099 error = ENOMEM; 1100 goto out; 1101 } 1102 } else { 1103 cuio_copyback((struct uio *)crp->crp_buf, 1104 crde->crd_skip, crde->crd_len, encbuf); 1105 } 1106 } 1107 1108 if (crda != NULL) { 1109 /* 1110 * Compute authentication tag after encryption. 1111 * This also handles the authentication only case. 1112 */ 1113 if (crde == NULL || (crde->crd_flags & CRD_F_ENCRYPT) != 0) 1114 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1115 1116 /* Copy back the authentication tag. */ 1117 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1118 if (m_copyback((struct mbuf *)crp->crp_buf, 1119 crda->crd_inject, ses->ses_hmac->taglen, tag, 1120 M_NOWAIT)) { 1121 error = ENOMEM; 1122 goto out; 1123 } 1124 } else { 1125 memcpy(crp->crp_mac, tag, ses->ses_hmac->taglen); 1126 } 1127 1128 explicit_bzero(tag, sizeof(tag)); 1129 } 1130 1131 out: 1132 explicit_bzero(buf, len); 1133 return error; 1134 } 1135 1136 void 1137 octcrypto_ghash_update_md(GHASH_CTX *ghash, uint8_t *src, size_t len) 1138 { 1139 CTASSERT(offsetof(GHASH_CTX, H) % 8 == 0); 1140 CTASSERT(offsetof(GHASH_CTX, S) % 8 == 0); 1141 1142 cop2_enable(); 1143 octcrypto_ghash_init((uint64_t *)ghash->H, (uint64_t *)ghash->S); 1144 octcrypto_ghash_update(src, len); 1145 octcrypto_ghash_finish((uint64_t *)ghash->S); 1146 cop2_disable(); 1147 } 1148