1 /* $OpenBSD: octcrypto.c,v 1.1 2018/04/09 13:46:15 visa Exp $ */ 2 3 /* 4 * Copyright (c) 2018 Visa Hankala 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Driver for the OCTEON cryptographic unit. 21 */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/atomic.h> 26 #include <sys/malloc.h> 27 #include <sys/mbuf.h> 28 #include <sys/pool.h> 29 #include <sys/tree.h> 30 31 #include <crypto/cryptodev.h> 32 #include <crypto/cryptosoft.h> 33 #include <crypto/xform.h> 34 35 #include <mips64/mips_cpu.h> 36 37 #include <machine/octeonvar.h> 38 39 /* Maximum number of dwords in hash IV. */ 40 #define MAX_IVNW 8 41 42 /* Number of dwords needed to cover `n' bytes. */ 43 #define ndwords(n) (roundup(n, 8) / (8)) 44 45 struct octcrypto_softc; 46 47 struct octcrypto_hmac { 48 void (*transform)(const void *, size_t); 49 void (*get_iv)(uint64_t *); 50 void (*set_iv)(const uint64_t *); 51 void (*clear)(void); 52 uint16_t blocklen; 53 uint16_t taglen; 54 uint16_t countlen; 55 }; 56 57 struct octcrypto_session { 58 uint32_t ses_sid; /* RB key, keep first */ 59 unsigned int ses_refs; 60 RBT_ENTRY(octrcypto_session) 61 ses_entry; 62 struct octcrypto_softc *ses_sc; 63 64 /* AES parameters */ 65 uint64_t ses_key[4]; 66 int ses_klen; 67 uint8_t ses_nonce[AESCTR_NONCESIZE]; 68 69 /* HMAC parameters */ 70 const struct octcrypto_hmac 71 *ses_hmac; 72 uint64_t ses_iiv[MAX_IVNW]; /* HMAC inner IV */ 73 uint64_t ses_oiv[MAX_IVNW]; /* HMAC outer IV */ 74 75 /* GHASH parameters */ 76 uint64_t ses_ghkey[2]; 77 78 struct swcr_data *ses_swd; 79 }; 80 81 struct octcrypto_cpu { 82 uint8_t *pcpu_buf; 83 size_t pcpu_buflen; 84 }; 85 86 struct octcrypto_softc { 87 struct device sc_dev; 88 int32_t sc_cid; 89 uint32_t sc_sid; 90 struct mutex sc_mtx; 91 RBT_HEAD(octcrypto_tree, octcrypto_session) 92 sc_sessions; 93 struct octcrypto_cpu sc_cpu[MAXCPUS]; 94 }; 95 96 int octcrypto_match(struct device *, void *, void *); 97 void octcrypto_attach(struct device *, struct device *, void *); 98 99 int octcrypto_newsession(uint32_t *, struct cryptoini *); 100 int octcrypto_freesession(uint64_t); 101 int octcrypto_process(struct cryptop *); 102 103 struct octcrypto_session * 104 octcrypto_get(struct octcrypto_softc *, uint32_t); 105 void octcrypto_put(struct octcrypto_session *); 106 107 void octcrypto_hmac(struct cryptodesc *, uint8_t *, size_t, 108 struct octcrypto_session *, uint64_t *); 109 int octcrypto_authenc_gmac(struct cryptop *, struct cryptodesc *, 110 struct cryptodesc *, struct octcrypto_session *); 111 int octcrypto_authenc_hmac(struct cryptop *, struct cryptodesc *, 112 struct cryptodesc *, struct octcrypto_session *); 113 int octcrypto_swauth(struct cryptop *, struct cryptodesc *, 114 struct swcr_data *, uint8_t *); 115 116 void octcrypto_ghash_update_md(GHASH_CTX *, uint8_t *, size_t); 117 118 void octcrypto_aes_clear(void); 119 void octcrypto_aes_cbc_dec(void *, size_t, const void *); 120 void octcrypto_aes_cbc_enc(void *, size_t, const void *); 121 void octcrypto_aes_ctr_enc(void *, size_t, const void *); 122 void octcrypto_aes_enc(uint64_t *); 123 void octcrypto_aes_set_key(const uint64_t *, int); 124 125 void octcrypto_ghash_finish(uint64_t *); 126 void octcrypto_ghash_init(const uint64_t *, const uint64_t *); 127 void octcrypto_ghash_update(const void *, size_t); 128 129 void octcrypto_hash_md5(const void *, size_t); 130 void octcrypto_hash_sha1(const void *, size_t); 131 void octcrypto_hash_sha256(const void *, size_t); 132 void octcrypto_hash_sha512(const void *, size_t); 133 void octcrypto_hash_clearn(void); 134 void octcrypto_hash_clearw(void); 135 void octcrypto_hash_get_ivn(uint64_t *); 136 void octcrypto_hash_get_ivw(uint64_t *); 137 void octcrypto_hash_set_ivn(const uint64_t *); 138 void octcrypto_hash_set_ivw(const uint64_t *); 139 140 const struct cfattach octcrypto_ca = { 141 sizeof(struct octcrypto_softc), octcrypto_match, octcrypto_attach 142 }; 143 144 struct cfdriver octcrypto_cd = { 145 NULL, "octcrypto", DV_DULL 146 }; 147 148 static const struct octcrypto_hmac hmac_md5_96 = { 149 .transform = octcrypto_hash_md5, 150 .get_iv = octcrypto_hash_get_ivn, 151 .set_iv = octcrypto_hash_set_ivn, 152 .clear = octcrypto_hash_clearn, 153 .blocklen = 64, 154 .taglen = 12, 155 .countlen = 8 156 }; 157 158 static const struct octcrypto_hmac hmac_sha1_96 = { 159 .transform = octcrypto_hash_sha1, 160 .get_iv = octcrypto_hash_get_ivn, 161 .set_iv = octcrypto_hash_set_ivn, 162 .clear = octcrypto_hash_clearn, 163 .blocklen = 64, 164 .taglen = 12, 165 .countlen = 8 166 }; 167 168 static const struct octcrypto_hmac hmac_sha2_256_128 = { 169 .transform = octcrypto_hash_sha256, 170 .get_iv = octcrypto_hash_get_ivn, 171 .set_iv = octcrypto_hash_set_ivn, 172 .clear = octcrypto_hash_clearn, 173 .blocklen = 64, 174 .taglen = 16, 175 .countlen = 8 176 }; 177 178 static const struct octcrypto_hmac hmac_sha2_384_192 = { 179 .transform = octcrypto_hash_sha512, 180 .get_iv = octcrypto_hash_get_ivw, 181 .set_iv = octcrypto_hash_set_ivw, 182 .clear = octcrypto_hash_clearw, 183 .blocklen = 128, 184 .taglen = 24, 185 .countlen = 16 186 }; 187 188 static const struct octcrypto_hmac hmac_sha2_512_256 = { 189 .transform = octcrypto_hash_sha512, 190 .get_iv = octcrypto_hash_get_ivw, 191 .set_iv = octcrypto_hash_set_ivw, 192 .clear = octcrypto_hash_clearw, 193 .blocklen = 128, 194 .taglen = 32, 195 .countlen = 16 196 }; 197 198 static struct pool octcryptopl; 199 static struct octcrypto_softc *octcrypto_sc; 200 201 static inline int 202 octcrypto_cmp(const struct octcrypto_session *a, 203 const struct octcrypto_session *b) 204 { 205 if (a->ses_sid < b->ses_sid) 206 return -1; 207 if (a->ses_sid > b->ses_sid) 208 return 1; 209 return 0; 210 } 211 212 RBT_PROTOTYPE(octcrypto_tree, octcrypto_session, sess_entry, octcrypto_cmp); 213 RBT_GENERATE(octcrypto_tree, octcrypto_session, ses_entry, octcrypto_cmp); 214 215 static inline void 216 cop2_enable(void) 217 { 218 setsr(getsr() | SR_COP_2_BIT); 219 } 220 221 static inline void 222 cop2_disable(void) 223 { 224 setsr(getsr() & ~SR_COP_2_BIT); 225 } 226 227 int 228 octcrypto_match(struct device *parent, void *match, void *aux) 229 { 230 return 1; 231 } 232 233 void 234 octcrypto_attach(struct device *parent, struct device *self, void *aux) 235 { 236 int algs[CRYPTO_ALGORITHM_MAX + 1]; 237 struct octcrypto_softc *sc = (struct octcrypto_softc *)self; 238 239 pool_init(&octcryptopl, sizeof(struct octcrypto_session), 16, 0, 0, 240 "octcrypto", NULL); 241 pool_setlowat(&octcryptopl, 2); 242 243 mtx_init(&sc->sc_mtx, IPL_VM); 244 RBT_INIT(octcrypto_tree, &sc->sc_sessions); 245 246 sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_MPSAFE); 247 if (sc->sc_cid < 0) { 248 printf(": could not get driver id\n"); 249 return; 250 } 251 252 printf("\n"); 253 254 memset(algs, 0, sizeof(algs)); 255 256 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED; 257 algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED; 258 algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED; 259 260 algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 261 algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 262 algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 263 algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 264 265 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 266 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 267 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 268 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 269 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 270 271 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 272 273 algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED; 274 275 octcrypto_sc = sc; 276 277 crypto_register(sc->sc_cid, algs, octcrypto_newsession, 278 octcrypto_freesession, octcrypto_process); 279 280 ghash_update = octcrypto_ghash_update_md; 281 } 282 283 struct octcrypto_session * 284 octcrypto_get(struct octcrypto_softc *sc, uint32_t sid) 285 { 286 struct octcrypto_session *ses; 287 288 mtx_enter(&sc->sc_mtx); 289 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 290 (struct octcrypto_session *)&sid); 291 if (ses != NULL) 292 atomic_inc_int(&ses->ses_refs); 293 mtx_leave(&sc->sc_mtx); 294 return ses; 295 } 296 297 void 298 octcrypto_put(struct octcrypto_session *ses) 299 { 300 struct auth_hash *axf; 301 struct swcr_data *swd; 302 303 if (atomic_dec_int_nv(&ses->ses_refs) > 0) 304 return; 305 306 if (ses->ses_swd != NULL) { 307 swd = ses->ses_swd; 308 axf = swd->sw_axf; 309 310 if (swd->sw_ictx != NULL) { 311 explicit_bzero(swd->sw_ictx, axf->ctxsize); 312 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize); 313 } 314 if (swd->sw_octx != NULL) { 315 explicit_bzero(swd->sw_octx, axf->ctxsize); 316 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize); 317 } 318 free(swd, M_CRYPTO_DATA, sizeof(*swd)); 319 } 320 321 explicit_bzero(ses, sizeof(*ses)); 322 pool_put(&octcryptopl, ses); 323 } 324 325 int 326 octcrypto_newsession(uint32_t *sidp, struct cryptoini *cri) 327 { 328 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 329 struct auth_hash *axf; 330 struct cryptoini *c; 331 const struct octcrypto_hmac *hmac = NULL; 332 struct octcrypto_softc *sc = octcrypto_sc; 333 struct octcrypto_session *ses = NULL; 334 struct swcr_data *swd; 335 uint8_t *bptr; 336 size_t klen; 337 int i; 338 uint32_t sid; 339 340 if (sidp == NULL || cri == NULL) 341 return EINVAL; 342 343 ses = pool_get(&octcryptopl, PR_NOWAIT | PR_ZERO); 344 if (ses == NULL) 345 return ENOMEM; 346 ses->ses_sc = sc; 347 ses->ses_refs = 1; 348 349 for (c = cri; c != NULL; c = c->cri_next) { 350 switch (c->cri_alg) { 351 case CRYPTO_AES_CBC: 352 ses->ses_klen = c->cri_klen / 8; 353 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 354 break; 355 356 case CRYPTO_AES_CTR: 357 case CRYPTO_AES_GCM_16: 358 case CRYPTO_AES_GMAC: 359 ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE; 360 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 361 memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen, 362 AESCTR_NONCESIZE); 363 break; 364 365 case CRYPTO_AES_128_GMAC: 366 case CRYPTO_AES_192_GMAC: 367 case CRYPTO_AES_256_GMAC: 368 cop2_enable(); 369 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 370 octcrypto_aes_enc(ses->ses_ghkey); 371 octcrypto_aes_clear(); 372 cop2_disable(); 373 break; 374 375 case CRYPTO_MD5_HMAC: 376 ses->ses_iiv[0] = 0x0123456789abcdefULL; 377 ses->ses_iiv[1] = 0xfedcba9876543210ULL; 378 ses->ses_hmac = &hmac_md5_96; 379 goto hwauthcommon; 380 381 case CRYPTO_SHA1_HMAC: 382 ses->ses_iiv[0] = 0x67452301efcdab89ULL; 383 ses->ses_iiv[1] = 0x98badcfe10325476ULL; 384 ses->ses_iiv[2] = 0xc3d2e1f000000000ULL; 385 ses->ses_hmac = &hmac_sha1_96; 386 goto hwauthcommon; 387 388 case CRYPTO_SHA2_256_HMAC: 389 ses->ses_iiv[0] = 0x6a09e667bb67ae85ULL; 390 ses->ses_iiv[1] = 0x3c6ef372a54ff53aULL; 391 ses->ses_iiv[2] = 0x510e527f9b05688cULL; 392 ses->ses_iiv[3] = 0x1f83d9ab5be0cd19ULL; 393 ses->ses_hmac = &hmac_sha2_256_128; 394 goto hwauthcommon; 395 396 case CRYPTO_SHA2_384_HMAC: 397 ses->ses_iiv[0] = 0xcbbb9d5dc1059ed8ULL; 398 ses->ses_iiv[1] = 0x629a292a367cd507ULL; 399 ses->ses_iiv[2] = 0x9159015a3070dd17ULL; 400 ses->ses_iiv[3] = 0x152fecd8f70e5939ULL; 401 ses->ses_iiv[4] = 0x67332667ffc00b31ULL; 402 ses->ses_iiv[5] = 0x8eb44a8768581511ULL; 403 ses->ses_iiv[6] = 0xdb0c2e0d64f98fa7ULL; 404 ses->ses_iiv[7] = 0x47b5481dbefa4fa4ULL; 405 ses->ses_hmac = &hmac_sha2_384_192; 406 goto hwauthcommon; 407 408 case CRYPTO_SHA2_512_HMAC: 409 ses->ses_iiv[0] = 0x6a09e667f3bcc908ULL; 410 ses->ses_iiv[1] = 0xbb67ae8584caa73bULL; 411 ses->ses_iiv[2] = 0x3c6ef372fe94f82bULL; 412 ses->ses_iiv[3] = 0xa54ff53a5f1d36f1ULL; 413 ses->ses_iiv[4] = 0x510e527fade682d1ULL; 414 ses->ses_iiv[5] = 0x9b05688c2b3e6c1fULL; 415 ses->ses_iiv[6] = 0x1f83d9abfb41bd6bULL; 416 ses->ses_iiv[7] = 0x5be0cd19137e2179ULL; 417 ses->ses_hmac = &hmac_sha2_512_256; 418 419 hwauthcommon: 420 memcpy(ses->ses_oiv, ses->ses_iiv, 421 sizeof(uint64_t) * MAX_IVNW); 422 423 bptr = (char *)block; 424 klen = c->cri_klen / 8; 425 hmac = ses->ses_hmac; 426 427 memcpy(bptr, c->cri_key, klen); 428 memset(bptr + klen, 0, hmac->blocklen - klen); 429 for (i = 0; i < hmac->blocklen; i++) 430 bptr[i] ^= HMAC_IPAD_VAL; 431 432 cop2_enable(); 433 hmac->set_iv(ses->ses_iiv); 434 hmac->transform(block, hmac->blocklen); 435 hmac->get_iv(ses->ses_iiv); 436 437 for (i = 0; i < hmac->blocklen; i++) 438 bptr[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 439 440 hmac->set_iv(ses->ses_oiv); 441 hmac->transform(block, hmac->blocklen); 442 hmac->get_iv(ses->ses_oiv); 443 hmac->clear(); 444 cop2_disable(); 445 446 explicit_bzero(block, hmac->blocklen); 447 break; 448 449 case CRYPTO_RIPEMD160_HMAC: 450 axf = &auth_hash_hmac_ripemd_160_96; 451 goto swauthcommon; 452 453 swauthcommon: 454 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 455 M_NOWAIT | M_ZERO); 456 if (swd == NULL) { 457 octcrypto_put(ses); 458 return ENOMEM; 459 } 460 ses->ses_swd = swd; 461 462 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 463 M_NOWAIT); 464 if (swd->sw_ictx == NULL) { 465 octcrypto_put(ses); 466 return ENOMEM; 467 } 468 469 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 470 M_NOWAIT); 471 if (swd->sw_octx == NULL) { 472 octcrypto_put(ses); 473 return ENOMEM; 474 } 475 476 for (i = 0; i < c->cri_klen / 8; i++) 477 c->cri_key[i] ^= HMAC_IPAD_VAL; 478 479 axf->Init(swd->sw_ictx); 480 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 481 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 482 axf->blocksize - (c->cri_klen / 8)); 483 484 for (i = 0; i < c->cri_klen / 8; i++) 485 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 486 HMAC_OPAD_VAL); 487 488 axf->Init(swd->sw_octx); 489 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 490 axf->Update(swd->sw_octx, hmac_opad_buffer, 491 axf->blocksize - (c->cri_klen / 8)); 492 493 for (i = 0; i < c->cri_klen / 8; i++) 494 c->cri_key[i] ^= HMAC_OPAD_VAL; 495 496 swd->sw_axf = axf; 497 swd->sw_alg = c->cri_alg; 498 499 break; 500 501 case CRYPTO_ESN: 502 /* nothing to do */ 503 break; 504 505 default: 506 octcrypto_put(ses); 507 return EINVAL; 508 } 509 } 510 511 mtx_enter(&sc->sc_mtx); 512 /* Find a free session ID. Assume there is one. */ 513 do { 514 sc->sc_sid++; 515 if (sc->sc_sid == 0) 516 sc->sc_sid = 1; 517 sid = sc->sc_sid; 518 } while (RBT_FIND(octcrypto_tree, &sc->sc_sessions, 519 (struct octcrypto_session *)&sid) != NULL); 520 ses->ses_sid = sid; 521 RBT_INSERT(octcrypto_tree, &sc->sc_sessions, ses); 522 mtx_leave(&sc->sc_mtx); 523 524 *sidp = ses->ses_sid; 525 return 0; 526 } 527 528 int 529 octcrypto_freesession(uint64_t tid) 530 { 531 struct octcrypto_softc *sc = octcrypto_sc; 532 struct octcrypto_session *ses; 533 uint32_t sid = (uint32_t)tid; 534 535 mtx_enter(&sc->sc_mtx); 536 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 537 (struct octcrypto_session *)&sid); 538 if (ses != NULL) 539 RBT_REMOVE(octcrypto_tree, &sc->sc_sessions, ses); 540 mtx_leave(&sc->sc_mtx); 541 542 if (ses == NULL) 543 return EINVAL; 544 545 octcrypto_put(ses); 546 547 return 0; 548 } 549 550 enum { 551 ALG_UNHANDLED, 552 ALG_AES, 553 ALG_AES_GHASH, 554 ALG_GMAC, 555 ALG_HMAC 556 }; 557 558 static int 559 alg_class(int alg) 560 { 561 switch (alg) { 562 case CRYPTO_AES_CBC: 563 case CRYPTO_AES_CTR: 564 return ALG_AES; 565 case CRYPTO_AES_GCM_16: 566 case CRYPTO_AES_GMAC: 567 return ALG_AES_GHASH; 568 case CRYPTO_AES_128_GMAC: 569 case CRYPTO_AES_192_GMAC: 570 case CRYPTO_AES_256_GMAC: 571 return ALG_GMAC; 572 case CRYPTO_MD5_HMAC: 573 case CRYPTO_SHA1_HMAC: 574 case CRYPTO_SHA2_256_HMAC: 575 case CRYPTO_SHA2_384_HMAC: 576 case CRYPTO_SHA2_512_HMAC: 577 return ALG_HMAC; 578 default: 579 return ALG_UNHANDLED; 580 } 581 } 582 583 int 584 octcrypto_process(struct cryptop *crp) 585 { 586 struct cryptodesc *crd, *crd2; 587 struct octcrypto_softc *sc = octcrypto_sc; 588 struct octcrypto_session *ses = NULL; 589 int alg, alg2; 590 int error = 0; 591 int i; 592 593 if (crp == NULL || crp->crp_callback == NULL) 594 return EINVAL; 595 596 KASSERT(crp->crp_ndesc >= 1); 597 598 ses = octcrypto_get(sc, (uint32_t)crp->crp_sid); 599 if (ses == NULL) { 600 error = EINVAL; 601 goto out; 602 } 603 604 if (crp->crp_ndesc == 2) { 605 crd = &crp->crp_desc[0]; 606 crd2 = &crp->crp_desc[1]; 607 alg = alg_class(crd->crd_alg); 608 alg2 = alg_class(crd2->crd_alg); 609 610 if ((alg == ALG_AES) && (alg2 == ALG_HMAC)) { 611 error = octcrypto_authenc_hmac(crp, crd, crd2, ses); 612 goto out; 613 } else if ((alg2 == ALG_AES) && (alg == ALG_HMAC)) { 614 error = octcrypto_authenc_hmac(crp, crd2, crd, ses); 615 goto out; 616 } else if ((alg == ALG_AES_GHASH) && (alg2 == ALG_GMAC)) { 617 error = octcrypto_authenc_gmac(crp, crd, crd2, ses); 618 goto out; 619 } else if ((alg2 == ALG_AES_GHASH) && (alg == ALG_GMAC)) { 620 error = octcrypto_authenc_gmac(crp, crd2, crd, ses); 621 goto out; 622 } 623 } 624 625 for (i = 0; i < crp->crp_ndesc; i++) { 626 crd = &crp->crp_desc[i]; 627 switch (crd->crd_alg) { 628 case CRYPTO_AES_CBC: 629 case CRYPTO_AES_CTR: 630 error = octcrypto_authenc_hmac(crp, crd, NULL, ses); 631 break; 632 633 case CRYPTO_MD5_HMAC: 634 case CRYPTO_SHA1_HMAC: 635 case CRYPTO_SHA2_256_HMAC: 636 case CRYPTO_SHA2_384_HMAC: 637 case CRYPTO_SHA2_512_HMAC: 638 error = octcrypto_authenc_hmac(crp, NULL, crd, ses); 639 break; 640 641 case CRYPTO_RIPEMD160_HMAC: 642 error = octcrypto_swauth(crp, crd, ses->ses_swd, 643 crp->crp_buf); 644 break; 645 646 default: 647 error = EINVAL; 648 break; 649 } 650 } 651 652 out: 653 if (ses != NULL) 654 octcrypto_put(ses); 655 crp->crp_etype = error; 656 crypto_done(crp); 657 return error; 658 } 659 660 int 661 octcrypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 662 struct swcr_data *sw, uint8_t *buf) 663 { 664 int type; 665 666 if (crp->crp_flags & CRYPTO_F_IMBUF) 667 type = CRYPTO_BUF_MBUF; 668 else 669 type = CRYPTO_BUF_IOV; 670 671 return swcr_authcompute(crp, crd, sw, buf, type); 672 } 673 674 int 675 octcrypto_authenc_gmac(struct cryptop *crp, struct cryptodesc *crde, 676 struct cryptodesc *crda, struct octcrypto_session *ses) 677 { 678 uint64_t block[ndwords(AESCTR_BLOCKSIZE)]; 679 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 680 uint64_t iv[ndwords(AESCTR_BLOCKSIZE)]; 681 uint64_t tag[ndwords(GMAC_BLOCK_LEN)]; 682 uint8_t *buf; 683 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 684 size_t aadlen; 685 size_t ivlen = 8; 686 size_t rlen; 687 int error = 0; 688 unsigned int iskip = 0; 689 unsigned int oskip = 0; 690 691 KASSERT(crda != NULL); 692 KASSERT(crde != NULL); 693 694 rlen = roundup(crde->crd_len, AESCTR_BLOCKSIZE); 695 if (rlen > pcpu->pcpu_buflen) { 696 if (pcpu->pcpu_buf != NULL) { 697 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 698 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 699 } 700 pcpu->pcpu_buflen = 0; 701 pcpu->pcpu_buf = malloc(rlen, M_DEVBUF, M_NOWAIT | M_ZERO); 702 if (pcpu->pcpu_buf == NULL) 703 return ENOMEM; 704 pcpu->pcpu_buflen = rlen; 705 } 706 buf = pcpu->pcpu_buf; 707 708 /* Prepare the IV. */ 709 if (crde->crd_flags & CRD_F_ENCRYPT) { 710 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 711 memcpy(iv, crde->crd_iv, ivlen); 712 else 713 arc4random_buf(iv, ivlen); 714 715 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 716 if (crp->crp_flags & CRYPTO_F_IMBUF) { 717 if (m_copyback((struct mbuf *)crp->crp_buf, 718 crde->crd_inject, ivlen, (uint8_t *)iv, 719 M_NOWAIT)) { 720 error = ENOMEM; 721 goto out; 722 } 723 } else { 724 cuio_copyback((struct uio *)crp->crp_buf, 725 crde->crd_inject, ivlen, (uint8_t *)iv); 726 } 727 } 728 } else { 729 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 730 memcpy(iv, crde->crd_iv, ivlen); 731 } else { 732 if (crp->crp_flags & CRYPTO_F_IMBUF) 733 m_copydata((struct mbuf *)crp->crp_buf, 734 crde->crd_inject, ivlen, (uint8_t *)iv); 735 else 736 cuio_copydata((struct uio *)crp->crp_buf, 737 crde->crd_inject, ivlen, (uint8_t *)iv); 738 } 739 } 740 741 memset(icb, 0, sizeof(icb)); 742 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 743 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE); 744 ((uint8_t *)icb)[AESCTR_BLOCKSIZE - 1] = 1; 745 746 /* Prepare the AAD. */ 747 aadlen = crda->crd_len; 748 if (crda->crd_flags & CRD_F_ESN) { 749 aadlen += 4; 750 if (crp->crp_flags & CRYPTO_F_IMBUF) 751 m_copydata((struct mbuf *)crp->crp_buf, 752 crda->crd_skip, 4, buf); 753 else 754 cuio_copydata((struct uio *)crp->crp_buf, 755 crda->crd_skip, 4, buf); 756 memcpy(buf + 4, crda->crd_esn, 4); 757 iskip = 4; 758 oskip = 8; 759 } 760 if (crp->crp_flags & CRYPTO_F_IMBUF) 761 m_copydata((struct mbuf *)crp->crp_buf, 762 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 763 else 764 cuio_copydata((struct uio *)crp->crp_buf, 765 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 766 767 cop2_enable(); 768 octcrypto_ghash_init(ses->ses_ghkey, NULL); 769 octcrypto_ghash_update(buf, roundup(aadlen, GMAC_BLOCK_LEN)); 770 cop2_disable(); 771 772 memset(buf, 0, aadlen); 773 774 /* Copy input to the working buffer. */ 775 if (crp->crp_flags & CRYPTO_F_IMBUF) 776 m_copydata((struct mbuf *)crp->crp_buf, crde->crd_skip, 777 crde->crd_len, buf); 778 else 779 cuio_copydata((struct uio *)crp->crp_buf, crde->crd_skip, 780 crde->crd_len, buf); 781 782 cop2_enable(); 783 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 784 785 switch (crde->crd_alg) { 786 case CRYPTO_AES_GCM_16: 787 if (crde->crd_flags & CRD_F_ENCRYPT) { 788 octcrypto_aes_ctr_enc(buf, rlen, icb); 789 memset(buf + crde->crd_len, 0, rlen - crde->crd_len); 790 octcrypto_ghash_update(buf, rlen); 791 } else { 792 octcrypto_ghash_update(buf, rlen); 793 octcrypto_aes_ctr_enc(buf, rlen, icb); 794 } 795 break; 796 797 case CRYPTO_AES_GMAC: 798 octcrypto_ghash_update(buf, rlen); 799 break; 800 } 801 802 block[0] = htobe64(aadlen * 8); 803 block[1] = htobe64(crde->crd_len * 8); 804 octcrypto_ghash_update(block, GMAC_BLOCK_LEN); 805 octcrypto_ghash_finish(tag); 806 807 block[0] = icb[0]; 808 block[1] = icb[1]; 809 octcrypto_aes_enc(block); 810 tag[0] ^= block[0]; 811 tag[1] ^= block[1]; 812 813 octcrypto_aes_clear(); 814 cop2_disable(); 815 816 /* Copy back the output. */ 817 if (crp->crp_flags & CRYPTO_F_IMBUF) { 818 if (m_copyback((struct mbuf *)crp->crp_buf, 819 crde->crd_skip, crde->crd_len, buf, M_NOWAIT)) { 820 error = ENOMEM; 821 goto out; 822 } 823 } else { 824 cuio_copyback((struct uio *)crp->crp_buf, 825 crde->crd_skip, crde->crd_len, buf); 826 } 827 828 /* Copy back the authentication tag. */ 829 if (crp->crp_flags & CRYPTO_F_IMBUF) { 830 if (m_copyback((struct mbuf *)crp->crp_buf, crda->crd_inject, 831 GMAC_DIGEST_LEN, tag, M_NOWAIT)) { 832 error = ENOMEM; 833 goto out; 834 } 835 } else { 836 memcpy(crp->crp_mac, tag, GMAC_DIGEST_LEN); 837 } 838 839 out: 840 explicit_bzero(buf, rlen); 841 explicit_bzero(icb, sizeof(icb)); 842 explicit_bzero(tag, sizeof(tag)); 843 844 return error; 845 } 846 847 void 848 octcrypto_hmac(struct cryptodesc *crda, uint8_t *buf, size_t len, 849 struct octcrypto_session *ses, uint64_t *res) 850 { 851 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 852 uint8_t *bptr = (uint8_t *)block; 853 const struct octcrypto_hmac *hmac = ses->ses_hmac; 854 size_t left; 855 856 cop2_enable(); 857 858 /* 859 * Compute the inner hash. 860 */ 861 862 hmac->set_iv(ses->ses_iiv); 863 hmac->transform(buf, len); 864 865 memset(block, 0, hmac->blocklen); 866 left = len & (hmac->blocklen - 1); 867 bptr[left] = 0x80; 868 if (left > 0) { 869 memcpy(block, buf + len - left, left); 870 871 if (roundup(left + 1, hmac->countlen) > 872 (hmac->blocklen - hmac->countlen)) { 873 hmac->transform(block, hmac->blocklen); 874 memset(block, 0, hmac->blocklen); 875 } 876 } 877 878 switch (crda->crd_alg) { 879 case CRYPTO_MD5_HMAC: 880 block[7] = htole64((64 + len) * 8); 881 break; 882 case CRYPTO_SHA1_HMAC: 883 case CRYPTO_SHA2_256_HMAC: 884 block[7] = htobe64((64 + len) * 8); 885 break; 886 case CRYPTO_SHA2_384_HMAC: 887 case CRYPTO_SHA2_512_HMAC: 888 block[15] = htobe64((128 + len) * 8); 889 break; 890 } 891 892 hmac->transform(block, hmac->blocklen); 893 894 /* 895 * Compute the outer hash. 896 */ 897 898 memset(block, 0, hmac->blocklen); 899 hmac->get_iv(block); 900 hmac->set_iv(ses->ses_oiv); 901 902 switch (crda->crd_alg) { 903 case CRYPTO_MD5_HMAC: 904 block[2] = htobe64(1ULL << 63); 905 block[7] = htole64((64 + 16) * 8); 906 break; 907 case CRYPTO_SHA1_HMAC: 908 block[2] |= htobe64(1ULL << 31); 909 block[7] = htobe64((64 + 20) * 8); 910 break; 911 case CRYPTO_SHA2_256_HMAC: 912 block[4] = htobe64(1ULL << 63); 913 block[7] = htobe64((64 + 32) * 8); 914 break; 915 case CRYPTO_SHA2_384_HMAC: 916 /* 917 * The computed digest is 512 bits long. 918 * It has to be truncated to 384 bits. 919 */ 920 block[6] = htobe64(1ULL << 63); 921 block[7] = 0; /* truncation */ 922 block[15] = htobe64((128 + 48) * 8); 923 break; 924 case CRYPTO_SHA2_512_HMAC: 925 block[8] = htobe64(1ULL << 63); 926 block[15] = htobe64((128 + 64) * 8); 927 break; 928 } 929 930 hmac->transform(block, hmac->blocklen); 931 hmac->get_iv(res); 932 hmac->clear(); 933 934 cop2_disable(); 935 936 explicit_bzero(block, sizeof(block)); 937 } 938 939 int 940 octcrypto_authenc_hmac(struct cryptop *crp, struct cryptodesc *crde, 941 struct cryptodesc *crda, struct octcrypto_session *ses) 942 { 943 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 944 uint64_t iv[ndwords(EALG_MAX_BLOCK_LEN)]; 945 uint64_t tag[ndwords(AALG_MAX_RESULT_LEN)]; 946 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 947 uint8_t *buf, *authbuf, *encbuf; 948 size_t authlen; 949 size_t buflen; 950 size_t len; 951 size_t skip; 952 off_t authskip = 0; 953 off_t encskip = 0; 954 int error = 0; 955 int ivlen; 956 957 if (crde != NULL && crda != NULL) { 958 skip = MIN(crde->crd_skip, crda->crd_skip); 959 len = MAX(crde->crd_skip + crde->crd_len, 960 crda->crd_skip + crda->crd_len) - skip; 961 962 if (crda->crd_skip < crde->crd_skip) 963 encskip = crde->crd_skip - crda->crd_skip; 964 else 965 authskip = crda->crd_skip - crde->crd_skip; 966 } else if (crde != NULL) { 967 skip = crde->crd_skip; 968 len = crde->crd_len; 969 } else { 970 KASSERT(crda != NULL); 971 972 skip = crda->crd_skip; 973 len = crda->crd_len; 974 } 975 976 buflen = len; 977 978 /* Reserve space for ESN. */ 979 if (crda != NULL && (crda->crd_flags & CRD_F_ESN) != 0) 980 buflen += 4; 981 982 buflen = roundup(buflen, EALG_MAX_BLOCK_LEN); 983 if (buflen > pcpu->pcpu_buflen) { 984 if (pcpu->pcpu_buf != NULL) { 985 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 986 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 987 } 988 pcpu->pcpu_buflen = 0; 989 pcpu->pcpu_buf = malloc(buflen, M_DEVBUF, M_NOWAIT | M_ZERO); 990 if (pcpu->pcpu_buf == NULL) 991 return ENOMEM; 992 pcpu->pcpu_buflen = buflen; 993 } 994 buf = pcpu->pcpu_buf; 995 996 authbuf = buf + authskip; 997 encbuf = buf + encskip; 998 999 /* Prepare the IV. */ 1000 if (crde != NULL) { 1001 /* CBC uses 16 bytes, CTR 8 bytes. */ 1002 ivlen = (crde->crd_alg == CRYPTO_AES_CBC) ? 16 : 8; 1003 1004 if (crde->crd_flags & CRD_F_ENCRYPT) { 1005 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 1006 memcpy(iv, crde->crd_iv, ivlen); 1007 else 1008 arc4random_buf(iv, ivlen); 1009 1010 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 1011 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1012 if (m_copyback( 1013 (struct mbuf *)crp->crp_buf, 1014 crde->crd_inject, ivlen, iv, 1015 M_NOWAIT)) { 1016 error = ENOMEM; 1017 goto out; 1018 } 1019 } else { 1020 cuio_copyback( 1021 (struct uio *)crp->crp_buf, 1022 crde->crd_inject, ivlen, iv); 1023 } 1024 } 1025 } else { 1026 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 1027 memcpy(iv, crde->crd_iv, ivlen); 1028 } else { 1029 if (crp->crp_flags & CRYPTO_F_IMBUF) 1030 m_copydata( 1031 (struct mbuf *)crp->crp_buf, 1032 crde->crd_inject, ivlen, 1033 (uint8_t *)iv); 1034 else 1035 cuio_copydata( 1036 (struct uio *)crp->crp_buf, 1037 crde->crd_inject, ivlen, 1038 (uint8_t *)iv); 1039 } 1040 } 1041 } 1042 1043 /* Copy input to the working buffer. */ 1044 if (crp->crp_flags & CRYPTO_F_IMBUF) 1045 m_copydata((struct mbuf *)crp->crp_buf, skip, len, buf); 1046 else 1047 cuio_copydata((struct uio *)crp->crp_buf, skip, len, buf); 1048 1049 /* If ESN is used, append it to the buffer. */ 1050 if (crda != NULL) { 1051 authlen = crda->crd_len; 1052 if (crda->crd_flags & CRD_F_ESN) { 1053 memcpy(buf + len, crda->crd_esn, 4); 1054 authlen += 4; 1055 } 1056 } 1057 1058 if (crde != NULL) { 1059 /* Compute authentication tag before decryption. */ 1060 if (crda != NULL && (crde->crd_flags & CRD_F_ENCRYPT) == 0) 1061 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1062 1063 /* Apply the cipher. */ 1064 switch (crde->crd_alg) { 1065 case CRYPTO_AES_CBC: 1066 cop2_enable(); 1067 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1068 if (crde->crd_flags & CRD_F_ENCRYPT) 1069 octcrypto_aes_cbc_enc(encbuf, crde->crd_len, 1070 iv); 1071 else 1072 octcrypto_aes_cbc_dec(encbuf, crde->crd_len, 1073 iv); 1074 octcrypto_aes_clear(); 1075 cop2_disable(); 1076 break; 1077 1078 case CRYPTO_AES_CTR: 1079 memset(icb, 0, sizeof(icb)); 1080 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 1081 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, 1082 AESCTR_IVSIZE); 1083 cop2_enable(); 1084 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1085 octcrypto_aes_ctr_enc(encbuf, crde->crd_len, icb); 1086 octcrypto_aes_clear(); 1087 cop2_disable(); 1088 explicit_bzero(icb, sizeof(icb)); 1089 break; 1090 } 1091 1092 /* Copy back the output. */ 1093 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1094 if (m_copyback((struct mbuf *)crp->crp_buf, 1095 crde->crd_skip, crde->crd_len, encbuf, M_NOWAIT)) { 1096 error = ENOMEM; 1097 goto out; 1098 } 1099 } else { 1100 cuio_copyback((struct uio *)crp->crp_buf, 1101 crde->crd_skip, crde->crd_len, encbuf); 1102 } 1103 } 1104 1105 if (crda != NULL) { 1106 /* 1107 * Compute authentication tag after encryption. 1108 * This also handles the authentication only case. 1109 */ 1110 if (crde == NULL || (crde->crd_flags & CRD_F_ENCRYPT) != 0) 1111 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1112 1113 /* Copy back the authentication tag. */ 1114 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1115 if (m_copyback((struct mbuf *)crp->crp_buf, 1116 crda->crd_inject, ses->ses_hmac->taglen, tag, 1117 M_NOWAIT)) { 1118 error = ENOMEM; 1119 goto out; 1120 } 1121 } else { 1122 memcpy(crp->crp_mac, tag, ses->ses_hmac->taglen); 1123 } 1124 1125 explicit_bzero(tag, sizeof(tag)); 1126 } 1127 1128 out: 1129 explicit_bzero(buf, len); 1130 return error; 1131 } 1132 1133 void 1134 octcrypto_ghash_update_md(GHASH_CTX *ghash, uint8_t *src, size_t len) 1135 { 1136 CTASSERT(offsetof(GHASH_CTX, H) % 8 == 0); 1137 CTASSERT(offsetof(GHASH_CTX, S) % 8 == 0); 1138 1139 cop2_enable(); 1140 octcrypto_ghash_init((uint64_t *)ghash->H, (uint64_t *)ghash->S); 1141 octcrypto_ghash_update(src, len); 1142 octcrypto_ghash_finish((uint64_t *)ghash->S); 1143 cop2_disable(); 1144 } 1145