1 /* $OpenBSD: octcrypto.c,v 1.3 2019/03/10 14:20:44 visa Exp $ */ 2 3 /* 4 * Copyright (c) 2018 Visa Hankala 5 * 6 * Permission to use, copy, modify, and/or distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Driver for the OCTEON cryptographic unit. 21 */ 22 23 #include <sys/param.h> 24 #include <sys/systm.h> 25 #include <sys/atomic.h> 26 #include <sys/malloc.h> 27 #include <sys/mbuf.h> 28 #include <sys/pool.h> 29 #include <sys/smr.h> 30 #include <sys/tree.h> 31 32 #include <crypto/cryptodev.h> 33 #include <crypto/cryptosoft.h> 34 #include <crypto/xform.h> 35 36 #include <mips64/mips_cpu.h> 37 38 #include <machine/octeonvar.h> 39 40 /* Maximum number of dwords in hash IV. */ 41 #define MAX_IVNW 8 42 43 /* Number of dwords needed to cover `n' bytes. */ 44 #define ndwords(n) (roundup(n, 8) / (8)) 45 46 struct octcrypto_softc; 47 48 struct octcrypto_hmac { 49 void (*transform)(const void *, size_t); 50 void (*get_iv)(uint64_t *); 51 void (*set_iv)(const uint64_t *); 52 void (*clear)(void); 53 uint16_t blocklen; 54 uint16_t taglen; 55 uint16_t countlen; 56 }; 57 58 struct octcrypto_session { 59 uint32_t ses_sid; /* RB key, keep first */ 60 RBT_ENTRY(octrcypto_session) 61 ses_entry; 62 struct octcrypto_softc *ses_sc; 63 struct smr_entry ses_smr; 64 65 /* AES parameters */ 66 uint64_t ses_key[4]; 67 int ses_klen; 68 uint8_t ses_nonce[AESCTR_NONCESIZE]; 69 70 /* HMAC parameters */ 71 const struct octcrypto_hmac 72 *ses_hmac; 73 uint64_t ses_iiv[MAX_IVNW]; /* HMAC inner IV */ 74 uint64_t ses_oiv[MAX_IVNW]; /* HMAC outer IV */ 75 76 /* GHASH parameters */ 77 uint64_t ses_ghkey[2]; 78 79 struct swcr_data *ses_swd; 80 }; 81 82 struct octcrypto_cpu { 83 uint8_t *pcpu_buf; 84 size_t pcpu_buflen; 85 }; 86 87 struct octcrypto_softc { 88 struct device sc_dev; 89 int32_t sc_cid; 90 uint32_t sc_sid; 91 struct mutex sc_mtx; 92 RBT_HEAD(octcrypto_tree, octcrypto_session) 93 sc_sessions; 94 struct octcrypto_cpu sc_cpu[MAXCPUS]; 95 }; 96 97 int octcrypto_match(struct device *, void *, void *); 98 void octcrypto_attach(struct device *, struct device *, void *); 99 100 int octcrypto_newsession(uint32_t *, struct cryptoini *); 101 int octcrypto_freesession(uint64_t); 102 int octcrypto_process(struct cryptop *); 103 104 struct octcrypto_session * 105 octcrypto_get(struct octcrypto_softc *, uint32_t); 106 void octcrypto_free(struct octcrypto_session *); 107 void octcrypto_free_smr(void *); 108 109 void octcrypto_hmac(struct cryptodesc *, uint8_t *, size_t, 110 struct octcrypto_session *, uint64_t *); 111 int octcrypto_authenc_gmac(struct cryptop *, struct cryptodesc *, 112 struct cryptodesc *, struct octcrypto_session *); 113 int octcrypto_authenc_hmac(struct cryptop *, struct cryptodesc *, 114 struct cryptodesc *, struct octcrypto_session *); 115 int octcrypto_swauth(struct cryptop *, struct cryptodesc *, 116 struct swcr_data *, uint8_t *); 117 118 void octcrypto_ghash_update_md(GHASH_CTX *, uint8_t *, size_t); 119 120 void octcrypto_aes_clear(void); 121 void octcrypto_aes_cbc_dec(void *, size_t, const void *); 122 void octcrypto_aes_cbc_enc(void *, size_t, const void *); 123 void octcrypto_aes_ctr_enc(void *, size_t, const void *); 124 void octcrypto_aes_enc(uint64_t *); 125 void octcrypto_aes_set_key(const uint64_t *, int); 126 127 void octcrypto_ghash_finish(uint64_t *); 128 void octcrypto_ghash_init(const uint64_t *, const uint64_t *); 129 void octcrypto_ghash_update(const void *, size_t); 130 131 void octcrypto_hash_md5(const void *, size_t); 132 void octcrypto_hash_sha1(const void *, size_t); 133 void octcrypto_hash_sha256(const void *, size_t); 134 void octcrypto_hash_sha512(const void *, size_t); 135 void octcrypto_hash_clearn(void); 136 void octcrypto_hash_clearw(void); 137 void octcrypto_hash_get_ivn(uint64_t *); 138 void octcrypto_hash_get_ivw(uint64_t *); 139 void octcrypto_hash_set_ivn(const uint64_t *); 140 void octcrypto_hash_set_ivw(const uint64_t *); 141 142 const struct cfattach octcrypto_ca = { 143 sizeof(struct octcrypto_softc), octcrypto_match, octcrypto_attach 144 }; 145 146 struct cfdriver octcrypto_cd = { 147 NULL, "octcrypto", DV_DULL 148 }; 149 150 static const struct octcrypto_hmac hmac_md5_96 = { 151 .transform = octcrypto_hash_md5, 152 .get_iv = octcrypto_hash_get_ivn, 153 .set_iv = octcrypto_hash_set_ivn, 154 .clear = octcrypto_hash_clearn, 155 .blocklen = 64, 156 .taglen = 12, 157 .countlen = 8 158 }; 159 160 static const struct octcrypto_hmac hmac_sha1_96 = { 161 .transform = octcrypto_hash_sha1, 162 .get_iv = octcrypto_hash_get_ivn, 163 .set_iv = octcrypto_hash_set_ivn, 164 .clear = octcrypto_hash_clearn, 165 .blocklen = 64, 166 .taglen = 12, 167 .countlen = 8 168 }; 169 170 static const struct octcrypto_hmac hmac_sha2_256_128 = { 171 .transform = octcrypto_hash_sha256, 172 .get_iv = octcrypto_hash_get_ivn, 173 .set_iv = octcrypto_hash_set_ivn, 174 .clear = octcrypto_hash_clearn, 175 .blocklen = 64, 176 .taglen = 16, 177 .countlen = 8 178 }; 179 180 static const struct octcrypto_hmac hmac_sha2_384_192 = { 181 .transform = octcrypto_hash_sha512, 182 .get_iv = octcrypto_hash_get_ivw, 183 .set_iv = octcrypto_hash_set_ivw, 184 .clear = octcrypto_hash_clearw, 185 .blocklen = 128, 186 .taglen = 24, 187 .countlen = 16 188 }; 189 190 static const struct octcrypto_hmac hmac_sha2_512_256 = { 191 .transform = octcrypto_hash_sha512, 192 .get_iv = octcrypto_hash_get_ivw, 193 .set_iv = octcrypto_hash_set_ivw, 194 .clear = octcrypto_hash_clearw, 195 .blocklen = 128, 196 .taglen = 32, 197 .countlen = 16 198 }; 199 200 static struct pool octcryptopl; 201 static struct octcrypto_softc *octcrypto_sc; 202 203 static inline int 204 octcrypto_cmp(const struct octcrypto_session *a, 205 const struct octcrypto_session *b) 206 { 207 if (a->ses_sid < b->ses_sid) 208 return -1; 209 if (a->ses_sid > b->ses_sid) 210 return 1; 211 return 0; 212 } 213 214 RBT_PROTOTYPE(octcrypto_tree, octcrypto_session, sess_entry, octcrypto_cmp); 215 RBT_GENERATE(octcrypto_tree, octcrypto_session, ses_entry, octcrypto_cmp); 216 217 static inline void 218 cop2_enable(void) 219 { 220 setsr(getsr() | SR_COP_2_BIT); 221 } 222 223 static inline void 224 cop2_disable(void) 225 { 226 setsr(getsr() & ~SR_COP_2_BIT); 227 } 228 229 int 230 octcrypto_match(struct device *parent, void *match, void *aux) 231 { 232 return 1; 233 } 234 235 void 236 octcrypto_attach(struct device *parent, struct device *self, void *aux) 237 { 238 int algs[CRYPTO_ALGORITHM_MAX + 1]; 239 struct octcrypto_softc *sc = (struct octcrypto_softc *)self; 240 241 pool_init(&octcryptopl, sizeof(struct octcrypto_session), 0, IPL_VM, 0, 242 "octcrypto", NULL); 243 pool_setlowat(&octcryptopl, 2); 244 245 mtx_init(&sc->sc_mtx, IPL_VM); 246 RBT_INIT(octcrypto_tree, &sc->sc_sessions); 247 248 sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_MPSAFE); 249 if (sc->sc_cid < 0) { 250 printf(": could not get driver id\n"); 251 return; 252 } 253 254 printf("\n"); 255 256 memset(algs, 0, sizeof(algs)); 257 258 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED; 259 algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED; 260 algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED; 261 262 algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 263 algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 264 algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 265 algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 266 267 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 268 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 269 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 270 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 271 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 272 273 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED; 274 275 algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED; 276 277 octcrypto_sc = sc; 278 279 crypto_register(sc->sc_cid, algs, octcrypto_newsession, 280 octcrypto_freesession, octcrypto_process); 281 282 ghash_update = octcrypto_ghash_update_md; 283 } 284 285 struct octcrypto_session * 286 octcrypto_get(struct octcrypto_softc *sc, uint32_t sid) 287 { 288 struct octcrypto_session *ses; 289 290 SMR_ASSERT_CRITICAL(); 291 292 mtx_enter(&sc->sc_mtx); 293 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 294 (struct octcrypto_session *)&sid); 295 mtx_leave(&sc->sc_mtx); 296 return ses; 297 } 298 299 void 300 octcrypto_free(struct octcrypto_session *ses) 301 { 302 struct auth_hash *axf; 303 struct swcr_data *swd; 304 305 if (ses->ses_swd != NULL) { 306 swd = ses->ses_swd; 307 axf = swd->sw_axf; 308 309 if (swd->sw_ictx != NULL) { 310 explicit_bzero(swd->sw_ictx, axf->ctxsize); 311 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize); 312 } 313 if (swd->sw_octx != NULL) { 314 explicit_bzero(swd->sw_octx, axf->ctxsize); 315 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize); 316 } 317 free(swd, M_CRYPTO_DATA, sizeof(*swd)); 318 } 319 320 explicit_bzero(ses, sizeof(*ses)); 321 pool_put(&octcryptopl, ses); 322 } 323 324 void 325 octcrypto_free_smr(void *arg) 326 { 327 struct octcrypto_session *ses = arg; 328 329 octcrypto_free(ses); 330 } 331 332 int 333 octcrypto_newsession(uint32_t *sidp, struct cryptoini *cri) 334 { 335 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 336 struct auth_hash *axf; 337 struct cryptoini *c; 338 const struct octcrypto_hmac *hmac = NULL; 339 struct octcrypto_softc *sc = octcrypto_sc; 340 struct octcrypto_session *ses = NULL; 341 struct swcr_data *swd; 342 uint8_t *bptr; 343 size_t klen; 344 int i; 345 uint32_t sid; 346 347 if (sidp == NULL || cri == NULL) 348 return EINVAL; 349 350 ses = pool_get(&octcryptopl, PR_NOWAIT | PR_ZERO); 351 if (ses == NULL) 352 return ENOMEM; 353 ses->ses_sc = sc; 354 smr_init(&ses->ses_smr); 355 356 for (c = cri; c != NULL; c = c->cri_next) { 357 switch (c->cri_alg) { 358 case CRYPTO_AES_CBC: 359 ses->ses_klen = c->cri_klen / 8; 360 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 361 break; 362 363 case CRYPTO_AES_CTR: 364 case CRYPTO_AES_GCM_16: 365 case CRYPTO_AES_GMAC: 366 ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE; 367 memcpy(ses->ses_key, c->cri_key, ses->ses_klen); 368 memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen, 369 AESCTR_NONCESIZE); 370 break; 371 372 case CRYPTO_AES_128_GMAC: 373 case CRYPTO_AES_192_GMAC: 374 case CRYPTO_AES_256_GMAC: 375 cop2_enable(); 376 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 377 octcrypto_aes_enc(ses->ses_ghkey); 378 octcrypto_aes_clear(); 379 cop2_disable(); 380 break; 381 382 case CRYPTO_MD5_HMAC: 383 ses->ses_iiv[0] = 0x0123456789abcdefULL; 384 ses->ses_iiv[1] = 0xfedcba9876543210ULL; 385 ses->ses_hmac = &hmac_md5_96; 386 goto hwauthcommon; 387 388 case CRYPTO_SHA1_HMAC: 389 ses->ses_iiv[0] = 0x67452301efcdab89ULL; 390 ses->ses_iiv[1] = 0x98badcfe10325476ULL; 391 ses->ses_iiv[2] = 0xc3d2e1f000000000ULL; 392 ses->ses_hmac = &hmac_sha1_96; 393 goto hwauthcommon; 394 395 case CRYPTO_SHA2_256_HMAC: 396 ses->ses_iiv[0] = 0x6a09e667bb67ae85ULL; 397 ses->ses_iiv[1] = 0x3c6ef372a54ff53aULL; 398 ses->ses_iiv[2] = 0x510e527f9b05688cULL; 399 ses->ses_iiv[3] = 0x1f83d9ab5be0cd19ULL; 400 ses->ses_hmac = &hmac_sha2_256_128; 401 goto hwauthcommon; 402 403 case CRYPTO_SHA2_384_HMAC: 404 ses->ses_iiv[0] = 0xcbbb9d5dc1059ed8ULL; 405 ses->ses_iiv[1] = 0x629a292a367cd507ULL; 406 ses->ses_iiv[2] = 0x9159015a3070dd17ULL; 407 ses->ses_iiv[3] = 0x152fecd8f70e5939ULL; 408 ses->ses_iiv[4] = 0x67332667ffc00b31ULL; 409 ses->ses_iiv[5] = 0x8eb44a8768581511ULL; 410 ses->ses_iiv[6] = 0xdb0c2e0d64f98fa7ULL; 411 ses->ses_iiv[7] = 0x47b5481dbefa4fa4ULL; 412 ses->ses_hmac = &hmac_sha2_384_192; 413 goto hwauthcommon; 414 415 case CRYPTO_SHA2_512_HMAC: 416 ses->ses_iiv[0] = 0x6a09e667f3bcc908ULL; 417 ses->ses_iiv[1] = 0xbb67ae8584caa73bULL; 418 ses->ses_iiv[2] = 0x3c6ef372fe94f82bULL; 419 ses->ses_iiv[3] = 0xa54ff53a5f1d36f1ULL; 420 ses->ses_iiv[4] = 0x510e527fade682d1ULL; 421 ses->ses_iiv[5] = 0x9b05688c2b3e6c1fULL; 422 ses->ses_iiv[6] = 0x1f83d9abfb41bd6bULL; 423 ses->ses_iiv[7] = 0x5be0cd19137e2179ULL; 424 ses->ses_hmac = &hmac_sha2_512_256; 425 426 hwauthcommon: 427 memcpy(ses->ses_oiv, ses->ses_iiv, 428 sizeof(uint64_t) * MAX_IVNW); 429 430 bptr = (char *)block; 431 klen = c->cri_klen / 8; 432 hmac = ses->ses_hmac; 433 434 memcpy(bptr, c->cri_key, klen); 435 memset(bptr + klen, 0, hmac->blocklen - klen); 436 for (i = 0; i < hmac->blocklen; i++) 437 bptr[i] ^= HMAC_IPAD_VAL; 438 439 cop2_enable(); 440 hmac->set_iv(ses->ses_iiv); 441 hmac->transform(block, hmac->blocklen); 442 hmac->get_iv(ses->ses_iiv); 443 444 for (i = 0; i < hmac->blocklen; i++) 445 bptr[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 446 447 hmac->set_iv(ses->ses_oiv); 448 hmac->transform(block, hmac->blocklen); 449 hmac->get_iv(ses->ses_oiv); 450 hmac->clear(); 451 cop2_disable(); 452 453 explicit_bzero(block, hmac->blocklen); 454 break; 455 456 case CRYPTO_RIPEMD160_HMAC: 457 axf = &auth_hash_hmac_ripemd_160_96; 458 goto swauthcommon; 459 460 swauthcommon: 461 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, 462 M_NOWAIT | M_ZERO); 463 if (swd == NULL) { 464 octcrypto_free(ses); 465 return ENOMEM; 466 } 467 ses->ses_swd = swd; 468 469 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 470 M_NOWAIT); 471 if (swd->sw_ictx == NULL) { 472 octcrypto_free(ses); 473 return ENOMEM; 474 } 475 476 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 477 M_NOWAIT); 478 if (swd->sw_octx == NULL) { 479 octcrypto_free(ses); 480 return ENOMEM; 481 } 482 483 for (i = 0; i < c->cri_klen / 8; i++) 484 c->cri_key[i] ^= HMAC_IPAD_VAL; 485 486 axf->Init(swd->sw_ictx); 487 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8); 488 axf->Update(swd->sw_ictx, hmac_ipad_buffer, 489 axf->blocksize - (c->cri_klen / 8)); 490 491 for (i = 0; i < c->cri_klen / 8; i++) 492 c->cri_key[i] ^= (HMAC_IPAD_VAL ^ 493 HMAC_OPAD_VAL); 494 495 axf->Init(swd->sw_octx); 496 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8); 497 axf->Update(swd->sw_octx, hmac_opad_buffer, 498 axf->blocksize - (c->cri_klen / 8)); 499 500 for (i = 0; i < c->cri_klen / 8; i++) 501 c->cri_key[i] ^= HMAC_OPAD_VAL; 502 503 swd->sw_axf = axf; 504 swd->sw_alg = c->cri_alg; 505 506 break; 507 508 case CRYPTO_ESN: 509 /* nothing to do */ 510 break; 511 512 default: 513 octcrypto_free(ses); 514 return EINVAL; 515 } 516 } 517 518 mtx_enter(&sc->sc_mtx); 519 /* Find a free session ID. Assume there is one. */ 520 do { 521 sc->sc_sid++; 522 if (sc->sc_sid == 0) 523 sc->sc_sid = 1; 524 sid = sc->sc_sid; 525 } while (RBT_FIND(octcrypto_tree, &sc->sc_sessions, 526 (struct octcrypto_session *)&sid) != NULL); 527 ses->ses_sid = sid; 528 RBT_INSERT(octcrypto_tree, &sc->sc_sessions, ses); 529 mtx_leave(&sc->sc_mtx); 530 531 *sidp = ses->ses_sid; 532 return 0; 533 } 534 535 int 536 octcrypto_freesession(uint64_t tid) 537 { 538 struct octcrypto_softc *sc = octcrypto_sc; 539 struct octcrypto_session *ses; 540 uint32_t sid = (uint32_t)tid; 541 542 mtx_enter(&sc->sc_mtx); 543 ses = RBT_FIND(octcrypto_tree, &sc->sc_sessions, 544 (struct octcrypto_session *)&sid); 545 if (ses != NULL) 546 RBT_REMOVE(octcrypto_tree, &sc->sc_sessions, ses); 547 mtx_leave(&sc->sc_mtx); 548 549 if (ses == NULL) 550 return EINVAL; 551 552 smr_call(&ses->ses_smr, octcrypto_free_smr, ses); 553 554 return 0; 555 } 556 557 enum { 558 ALG_UNHANDLED, 559 ALG_AES, 560 ALG_AES_GHASH, 561 ALG_GMAC, 562 ALG_HMAC 563 }; 564 565 static int 566 alg_class(int alg) 567 { 568 switch (alg) { 569 case CRYPTO_AES_CBC: 570 case CRYPTO_AES_CTR: 571 return ALG_AES; 572 case CRYPTO_AES_GCM_16: 573 case CRYPTO_AES_GMAC: 574 return ALG_AES_GHASH; 575 case CRYPTO_AES_128_GMAC: 576 case CRYPTO_AES_192_GMAC: 577 case CRYPTO_AES_256_GMAC: 578 return ALG_GMAC; 579 case CRYPTO_MD5_HMAC: 580 case CRYPTO_SHA1_HMAC: 581 case CRYPTO_SHA2_256_HMAC: 582 case CRYPTO_SHA2_384_HMAC: 583 case CRYPTO_SHA2_512_HMAC: 584 return ALG_HMAC; 585 default: 586 return ALG_UNHANDLED; 587 } 588 } 589 590 int 591 octcrypto_process(struct cryptop *crp) 592 { 593 struct cryptodesc *crd, *crd2; 594 struct octcrypto_softc *sc = octcrypto_sc; 595 struct octcrypto_session *ses = NULL; 596 int alg, alg2; 597 int error = 0; 598 int i; 599 600 if (crp == NULL || crp->crp_callback == NULL) 601 return EINVAL; 602 603 KASSERT(crp->crp_ndesc >= 1); 604 605 smr_read_enter(); 606 ses = octcrypto_get(sc, (uint32_t)crp->crp_sid); 607 if (ses == NULL) { 608 error = EINVAL; 609 goto out; 610 } 611 612 if (crp->crp_ndesc == 2) { 613 crd = &crp->crp_desc[0]; 614 crd2 = &crp->crp_desc[1]; 615 alg = alg_class(crd->crd_alg); 616 alg2 = alg_class(crd2->crd_alg); 617 618 if ((alg == ALG_AES) && (alg2 == ALG_HMAC)) { 619 error = octcrypto_authenc_hmac(crp, crd, crd2, ses); 620 goto out; 621 } else if ((alg2 == ALG_AES) && (alg == ALG_HMAC)) { 622 error = octcrypto_authenc_hmac(crp, crd2, crd, ses); 623 goto out; 624 } else if ((alg == ALG_AES_GHASH) && (alg2 == ALG_GMAC)) { 625 error = octcrypto_authenc_gmac(crp, crd, crd2, ses); 626 goto out; 627 } else if ((alg2 == ALG_AES_GHASH) && (alg == ALG_GMAC)) { 628 error = octcrypto_authenc_gmac(crp, crd2, crd, ses); 629 goto out; 630 } 631 } 632 633 for (i = 0; i < crp->crp_ndesc; i++) { 634 crd = &crp->crp_desc[i]; 635 switch (crd->crd_alg) { 636 case CRYPTO_AES_CBC: 637 case CRYPTO_AES_CTR: 638 error = octcrypto_authenc_hmac(crp, crd, NULL, ses); 639 break; 640 641 case CRYPTO_MD5_HMAC: 642 case CRYPTO_SHA1_HMAC: 643 case CRYPTO_SHA2_256_HMAC: 644 case CRYPTO_SHA2_384_HMAC: 645 case CRYPTO_SHA2_512_HMAC: 646 error = octcrypto_authenc_hmac(crp, NULL, crd, ses); 647 break; 648 649 case CRYPTO_RIPEMD160_HMAC: 650 error = octcrypto_swauth(crp, crd, ses->ses_swd, 651 crp->crp_buf); 652 break; 653 654 default: 655 error = EINVAL; 656 break; 657 } 658 } 659 660 out: 661 smr_read_leave(); 662 663 crp->crp_etype = error; 664 crypto_done(crp); 665 return error; 666 } 667 668 int 669 octcrypto_swauth(struct cryptop *crp, struct cryptodesc *crd, 670 struct swcr_data *sw, uint8_t *buf) 671 { 672 int type; 673 674 if (crp->crp_flags & CRYPTO_F_IMBUF) 675 type = CRYPTO_BUF_MBUF; 676 else 677 type = CRYPTO_BUF_IOV; 678 679 return swcr_authcompute(crp, crd, sw, buf, type); 680 } 681 682 int 683 octcrypto_authenc_gmac(struct cryptop *crp, struct cryptodesc *crde, 684 struct cryptodesc *crda, struct octcrypto_session *ses) 685 { 686 uint64_t block[ndwords(AESCTR_BLOCKSIZE)]; 687 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 688 uint64_t iv[ndwords(AESCTR_BLOCKSIZE)]; 689 uint64_t tag[ndwords(GMAC_BLOCK_LEN)]; 690 uint8_t *buf; 691 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 692 size_t aadlen; 693 size_t ivlen = 8; 694 size_t rlen; 695 int error = 0; 696 unsigned int iskip = 0; 697 unsigned int oskip = 0; 698 699 KASSERT(crda != NULL); 700 KASSERT(crde != NULL); 701 702 rlen = roundup(crde->crd_len, AESCTR_BLOCKSIZE); 703 if (rlen > pcpu->pcpu_buflen) { 704 if (pcpu->pcpu_buf != NULL) { 705 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 706 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 707 } 708 pcpu->pcpu_buflen = 0; 709 pcpu->pcpu_buf = malloc(rlen, M_DEVBUF, M_NOWAIT | M_ZERO); 710 if (pcpu->pcpu_buf == NULL) 711 return ENOMEM; 712 pcpu->pcpu_buflen = rlen; 713 } 714 buf = pcpu->pcpu_buf; 715 716 /* Prepare the IV. */ 717 if (crde->crd_flags & CRD_F_ENCRYPT) { 718 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 719 memcpy(iv, crde->crd_iv, ivlen); 720 else 721 arc4random_buf(iv, ivlen); 722 723 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 724 if (crp->crp_flags & CRYPTO_F_IMBUF) { 725 if (m_copyback((struct mbuf *)crp->crp_buf, 726 crde->crd_inject, ivlen, (uint8_t *)iv, 727 M_NOWAIT)) { 728 error = ENOMEM; 729 goto out; 730 } 731 } else { 732 cuio_copyback((struct uio *)crp->crp_buf, 733 crde->crd_inject, ivlen, (uint8_t *)iv); 734 } 735 } 736 } else { 737 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 738 memcpy(iv, crde->crd_iv, ivlen); 739 } else { 740 if (crp->crp_flags & CRYPTO_F_IMBUF) 741 m_copydata((struct mbuf *)crp->crp_buf, 742 crde->crd_inject, ivlen, (uint8_t *)iv); 743 else 744 cuio_copydata((struct uio *)crp->crp_buf, 745 crde->crd_inject, ivlen, (uint8_t *)iv); 746 } 747 } 748 749 memset(icb, 0, sizeof(icb)); 750 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 751 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE); 752 ((uint8_t *)icb)[AESCTR_BLOCKSIZE - 1] = 1; 753 754 /* Prepare the AAD. */ 755 aadlen = crda->crd_len; 756 if (crda->crd_flags & CRD_F_ESN) { 757 aadlen += 4; 758 if (crp->crp_flags & CRYPTO_F_IMBUF) 759 m_copydata((struct mbuf *)crp->crp_buf, 760 crda->crd_skip, 4, buf); 761 else 762 cuio_copydata((struct uio *)crp->crp_buf, 763 crda->crd_skip, 4, buf); 764 memcpy(buf + 4, crda->crd_esn, 4); 765 iskip = 4; 766 oskip = 8; 767 } 768 if (crp->crp_flags & CRYPTO_F_IMBUF) 769 m_copydata((struct mbuf *)crp->crp_buf, 770 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 771 else 772 cuio_copydata((struct uio *)crp->crp_buf, 773 crda->crd_skip + iskip, crda->crd_len - iskip, buf + oskip); 774 775 cop2_enable(); 776 octcrypto_ghash_init(ses->ses_ghkey, NULL); 777 octcrypto_ghash_update(buf, roundup(aadlen, GMAC_BLOCK_LEN)); 778 cop2_disable(); 779 780 memset(buf, 0, aadlen); 781 782 /* Copy input to the working buffer. */ 783 if (crp->crp_flags & CRYPTO_F_IMBUF) 784 m_copydata((struct mbuf *)crp->crp_buf, crde->crd_skip, 785 crde->crd_len, buf); 786 else 787 cuio_copydata((struct uio *)crp->crp_buf, crde->crd_skip, 788 crde->crd_len, buf); 789 790 cop2_enable(); 791 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 792 793 switch (crde->crd_alg) { 794 case CRYPTO_AES_GCM_16: 795 if (crde->crd_flags & CRD_F_ENCRYPT) { 796 octcrypto_aes_ctr_enc(buf, rlen, icb); 797 memset(buf + crde->crd_len, 0, rlen - crde->crd_len); 798 octcrypto_ghash_update(buf, rlen); 799 } else { 800 octcrypto_ghash_update(buf, rlen); 801 octcrypto_aes_ctr_enc(buf, rlen, icb); 802 } 803 break; 804 805 case CRYPTO_AES_GMAC: 806 octcrypto_ghash_update(buf, rlen); 807 break; 808 } 809 810 block[0] = htobe64(aadlen * 8); 811 block[1] = htobe64(crde->crd_len * 8); 812 octcrypto_ghash_update(block, GMAC_BLOCK_LEN); 813 octcrypto_ghash_finish(tag); 814 815 block[0] = icb[0]; 816 block[1] = icb[1]; 817 octcrypto_aes_enc(block); 818 tag[0] ^= block[0]; 819 tag[1] ^= block[1]; 820 821 octcrypto_aes_clear(); 822 cop2_disable(); 823 824 /* Copy back the output. */ 825 if (crp->crp_flags & CRYPTO_F_IMBUF) { 826 if (m_copyback((struct mbuf *)crp->crp_buf, 827 crde->crd_skip, crde->crd_len, buf, M_NOWAIT)) { 828 error = ENOMEM; 829 goto out; 830 } 831 } else { 832 cuio_copyback((struct uio *)crp->crp_buf, 833 crde->crd_skip, crde->crd_len, buf); 834 } 835 836 /* Copy back the authentication tag. */ 837 if (crp->crp_flags & CRYPTO_F_IMBUF) { 838 if (m_copyback((struct mbuf *)crp->crp_buf, crda->crd_inject, 839 GMAC_DIGEST_LEN, tag, M_NOWAIT)) { 840 error = ENOMEM; 841 goto out; 842 } 843 } else { 844 memcpy(crp->crp_mac, tag, GMAC_DIGEST_LEN); 845 } 846 847 out: 848 explicit_bzero(buf, rlen); 849 explicit_bzero(icb, sizeof(icb)); 850 explicit_bzero(tag, sizeof(tag)); 851 852 return error; 853 } 854 855 void 856 octcrypto_hmac(struct cryptodesc *crda, uint8_t *buf, size_t len, 857 struct octcrypto_session *ses, uint64_t *res) 858 { 859 uint64_t block[ndwords(HMAC_MAX_BLOCK_LEN)]; 860 uint8_t *bptr = (uint8_t *)block; 861 const struct octcrypto_hmac *hmac = ses->ses_hmac; 862 size_t left; 863 864 cop2_enable(); 865 866 /* 867 * Compute the inner hash. 868 */ 869 870 hmac->set_iv(ses->ses_iiv); 871 hmac->transform(buf, len); 872 873 memset(block, 0, hmac->blocklen); 874 left = len & (hmac->blocklen - 1); 875 bptr[left] = 0x80; 876 if (left > 0) { 877 memcpy(block, buf + len - left, left); 878 879 if (roundup(left + 1, hmac->countlen) > 880 (hmac->blocklen - hmac->countlen)) { 881 hmac->transform(block, hmac->blocklen); 882 memset(block, 0, hmac->blocklen); 883 } 884 } 885 886 switch (crda->crd_alg) { 887 case CRYPTO_MD5_HMAC: 888 block[7] = htole64((64 + len) * 8); 889 break; 890 case CRYPTO_SHA1_HMAC: 891 case CRYPTO_SHA2_256_HMAC: 892 block[7] = htobe64((64 + len) * 8); 893 break; 894 case CRYPTO_SHA2_384_HMAC: 895 case CRYPTO_SHA2_512_HMAC: 896 block[15] = htobe64((128 + len) * 8); 897 break; 898 } 899 900 hmac->transform(block, hmac->blocklen); 901 902 /* 903 * Compute the outer hash. 904 */ 905 906 memset(block, 0, hmac->blocklen); 907 hmac->get_iv(block); 908 hmac->set_iv(ses->ses_oiv); 909 910 switch (crda->crd_alg) { 911 case CRYPTO_MD5_HMAC: 912 block[2] = htobe64(1ULL << 63); 913 block[7] = htole64((64 + 16) * 8); 914 break; 915 case CRYPTO_SHA1_HMAC: 916 block[2] |= htobe64(1ULL << 31); 917 block[7] = htobe64((64 + 20) * 8); 918 break; 919 case CRYPTO_SHA2_256_HMAC: 920 block[4] = htobe64(1ULL << 63); 921 block[7] = htobe64((64 + 32) * 8); 922 break; 923 case CRYPTO_SHA2_384_HMAC: 924 /* 925 * The computed digest is 512 bits long. 926 * It has to be truncated to 384 bits. 927 */ 928 block[6] = htobe64(1ULL << 63); 929 block[7] = 0; /* truncation */ 930 block[15] = htobe64((128 + 48) * 8); 931 break; 932 case CRYPTO_SHA2_512_HMAC: 933 block[8] = htobe64(1ULL << 63); 934 block[15] = htobe64((128 + 64) * 8); 935 break; 936 } 937 938 hmac->transform(block, hmac->blocklen); 939 hmac->get_iv(res); 940 hmac->clear(); 941 942 cop2_disable(); 943 944 explicit_bzero(block, sizeof(block)); 945 } 946 947 int 948 octcrypto_authenc_hmac(struct cryptop *crp, struct cryptodesc *crde, 949 struct cryptodesc *crda, struct octcrypto_session *ses) 950 { 951 uint64_t icb[ndwords(AESCTR_BLOCKSIZE)]; 952 uint64_t iv[ndwords(EALG_MAX_BLOCK_LEN)]; 953 uint64_t tag[ndwords(AALG_MAX_RESULT_LEN)]; 954 struct octcrypto_cpu *pcpu = &ses->ses_sc->sc_cpu[cpu_number()]; 955 uint8_t *buf, *authbuf, *encbuf; 956 size_t authlen; 957 size_t buflen; 958 size_t len; 959 size_t skip; 960 off_t authskip = 0; 961 off_t encskip = 0; 962 int error = 0; 963 int ivlen; 964 965 if (crde != NULL && crda != NULL) { 966 skip = MIN(crde->crd_skip, crda->crd_skip); 967 len = MAX(crde->crd_skip + crde->crd_len, 968 crda->crd_skip + crda->crd_len) - skip; 969 970 if (crda->crd_skip < crde->crd_skip) 971 encskip = crde->crd_skip - crda->crd_skip; 972 else 973 authskip = crda->crd_skip - crde->crd_skip; 974 } else if (crde != NULL) { 975 skip = crde->crd_skip; 976 len = crde->crd_len; 977 } else { 978 KASSERT(crda != NULL); 979 980 skip = crda->crd_skip; 981 len = crda->crd_len; 982 } 983 984 buflen = len; 985 986 /* Reserve space for ESN. */ 987 if (crda != NULL && (crda->crd_flags & CRD_F_ESN) != 0) 988 buflen += 4; 989 990 buflen = roundup(buflen, EALG_MAX_BLOCK_LEN); 991 if (buflen > pcpu->pcpu_buflen) { 992 if (pcpu->pcpu_buf != NULL) { 993 explicit_bzero(pcpu->pcpu_buf, pcpu->pcpu_buflen); 994 free(pcpu->pcpu_buf, M_DEVBUF, pcpu->pcpu_buflen); 995 } 996 pcpu->pcpu_buflen = 0; 997 pcpu->pcpu_buf = malloc(buflen, M_DEVBUF, M_NOWAIT | M_ZERO); 998 if (pcpu->pcpu_buf == NULL) 999 return ENOMEM; 1000 pcpu->pcpu_buflen = buflen; 1001 } 1002 buf = pcpu->pcpu_buf; 1003 1004 authbuf = buf + authskip; 1005 encbuf = buf + encskip; 1006 1007 /* Prepare the IV. */ 1008 if (crde != NULL) { 1009 /* CBC uses 16 bytes, CTR 8 bytes. */ 1010 ivlen = (crde->crd_alg == CRYPTO_AES_CBC) ? 16 : 8; 1011 1012 if (crde->crd_flags & CRD_F_ENCRYPT) { 1013 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 1014 memcpy(iv, crde->crd_iv, ivlen); 1015 else 1016 arc4random_buf(iv, ivlen); 1017 1018 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) { 1019 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1020 if (m_copyback( 1021 (struct mbuf *)crp->crp_buf, 1022 crde->crd_inject, ivlen, iv, 1023 M_NOWAIT)) { 1024 error = ENOMEM; 1025 goto out; 1026 } 1027 } else { 1028 cuio_copyback( 1029 (struct uio *)crp->crp_buf, 1030 crde->crd_inject, ivlen, iv); 1031 } 1032 } 1033 } else { 1034 if (crde->crd_flags & CRD_F_IV_EXPLICIT) { 1035 memcpy(iv, crde->crd_iv, ivlen); 1036 } else { 1037 if (crp->crp_flags & CRYPTO_F_IMBUF) 1038 m_copydata( 1039 (struct mbuf *)crp->crp_buf, 1040 crde->crd_inject, ivlen, 1041 (uint8_t *)iv); 1042 else 1043 cuio_copydata( 1044 (struct uio *)crp->crp_buf, 1045 crde->crd_inject, ivlen, 1046 (uint8_t *)iv); 1047 } 1048 } 1049 } 1050 1051 /* Copy input to the working buffer. */ 1052 if (crp->crp_flags & CRYPTO_F_IMBUF) 1053 m_copydata((struct mbuf *)crp->crp_buf, skip, len, buf); 1054 else 1055 cuio_copydata((struct uio *)crp->crp_buf, skip, len, buf); 1056 1057 /* If ESN is used, append it to the buffer. */ 1058 if (crda != NULL) { 1059 authlen = crda->crd_len; 1060 if (crda->crd_flags & CRD_F_ESN) { 1061 memcpy(buf + len, crda->crd_esn, 4); 1062 authlen += 4; 1063 } 1064 } 1065 1066 if (crde != NULL) { 1067 /* Compute authentication tag before decryption. */ 1068 if (crda != NULL && (crde->crd_flags & CRD_F_ENCRYPT) == 0) 1069 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1070 1071 /* Apply the cipher. */ 1072 switch (crde->crd_alg) { 1073 case CRYPTO_AES_CBC: 1074 cop2_enable(); 1075 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1076 if (crde->crd_flags & CRD_F_ENCRYPT) 1077 octcrypto_aes_cbc_enc(encbuf, crde->crd_len, 1078 iv); 1079 else 1080 octcrypto_aes_cbc_dec(encbuf, crde->crd_len, 1081 iv); 1082 octcrypto_aes_clear(); 1083 cop2_disable(); 1084 break; 1085 1086 case CRYPTO_AES_CTR: 1087 memset(icb, 0, sizeof(icb)); 1088 memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE); 1089 memcpy((uint8_t *)icb + AESCTR_NONCESIZE, iv, 1090 AESCTR_IVSIZE); 1091 cop2_enable(); 1092 octcrypto_aes_set_key(ses->ses_key, ses->ses_klen); 1093 octcrypto_aes_ctr_enc(encbuf, crde->crd_len, icb); 1094 octcrypto_aes_clear(); 1095 cop2_disable(); 1096 explicit_bzero(icb, sizeof(icb)); 1097 break; 1098 } 1099 1100 /* Copy back the output. */ 1101 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1102 if (m_copyback((struct mbuf *)crp->crp_buf, 1103 crde->crd_skip, crde->crd_len, encbuf, M_NOWAIT)) { 1104 error = ENOMEM; 1105 goto out; 1106 } 1107 } else { 1108 cuio_copyback((struct uio *)crp->crp_buf, 1109 crde->crd_skip, crde->crd_len, encbuf); 1110 } 1111 } 1112 1113 if (crda != NULL) { 1114 /* 1115 * Compute authentication tag after encryption. 1116 * This also handles the authentication only case. 1117 */ 1118 if (crde == NULL || (crde->crd_flags & CRD_F_ENCRYPT) != 0) 1119 octcrypto_hmac(crda, authbuf, authlen, ses, tag); 1120 1121 /* Copy back the authentication tag. */ 1122 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1123 if (m_copyback((struct mbuf *)crp->crp_buf, 1124 crda->crd_inject, ses->ses_hmac->taglen, tag, 1125 M_NOWAIT)) { 1126 error = ENOMEM; 1127 goto out; 1128 } 1129 } else { 1130 memcpy(crp->crp_mac, tag, ses->ses_hmac->taglen); 1131 } 1132 1133 explicit_bzero(tag, sizeof(tag)); 1134 } 1135 1136 out: 1137 explicit_bzero(buf, len); 1138 return error; 1139 } 1140 1141 void 1142 octcrypto_ghash_update_md(GHASH_CTX *ghash, uint8_t *src, size_t len) 1143 { 1144 CTASSERT(offsetof(GHASH_CTX, H) % 8 == 0); 1145 CTASSERT(offsetof(GHASH_CTX, S) % 8 == 0); 1146 1147 cop2_enable(); 1148 octcrypto_ghash_init((uint64_t *)ghash->H, (uint64_t *)ghash->S); 1149 octcrypto_ghash_update(src, len); 1150 octcrypto_ghash_finish((uint64_t *)ghash->S); 1151 cop2_disable(); 1152 } 1153