1 /* $NetBSD: xform_esp.c,v 1.100 2020/06/30 04:14:55 riastradh Exp $ */ 2 /* $FreeBSD: xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */ 3 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ 4 5 /* 6 * The authors of this code are John Ioannidis (ji@tla.org), 7 * Angelos D. Keromytis (kermit@csd.uch.gr) and 8 * Niels Provos (provos@physnet.uni-hamburg.de). 9 * 10 * The original version of this code was written by John Ioannidis 11 * for BSD/OS in Athens, Greece, in November 1995. 12 * 13 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, 14 * by Angelos D. Keromytis. 15 * 16 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis 17 * and Niels Provos. 18 * 19 * Additional features in 1999 by Angelos D. Keromytis. 20 * 21 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, 22 * Angelos D. Keromytis and Niels Provos. 23 * Copyright (c) 2001 Angelos D. Keromytis. 24 * 25 * Permission to use, copy, and modify this software with or without fee 26 * is hereby granted, provided that this entire notice is included in 27 * all copies of any software which is or includes a copy or 28 * modification of this software. 29 * You may use this code under the GNU public license if you so wish. Please 30 * contribute changes back to the authors under this freer than GPL license 31 * so that we may further the use of strong encryption without limitations to 32 * all. 33 * 34 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 35 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 36 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 37 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 38 * PURPOSE. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.100 2020/06/30 04:14:55 riastradh Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_inet.h" 46 #include "opt_ipsec.h" 47 #endif 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/mbuf.h> 52 #include <sys/socket.h> 53 #include <sys/syslog.h> 54 #include <sys/kernel.h> 55 #include <sys/sysctl.h> 56 #include <sys/cprng.h> 57 #include <sys/pool.h> 58 #include <sys/pserialize.h> 59 60 #include <net/if.h> 61 62 #include <netinet/in.h> 63 #include <netinet/in_systm.h> 64 #include <netinet/ip.h> 65 #include <netinet/ip_ecn.h> 66 #include <netinet/ip6.h> 67 68 #include <net/route.h> 69 #include <netipsec/ipsec.h> 70 #include <netipsec/ipsec_private.h> 71 #include <netipsec/ah.h> 72 #include <netipsec/ah_var.h> 73 #include <netipsec/esp.h> 74 #include <netipsec/esp_var.h> 75 #include <netipsec/xform.h> 76 77 #ifdef INET6 78 #include <netinet6/ip6_var.h> 79 #include <netipsec/ipsec6.h> 80 #endif 81 82 #include <netipsec/key.h> 83 #include <netipsec/key_debug.h> 84 85 #include <opencrypto/cryptodev.h> 86 87 percpu_t *espstat_percpu; 88 89 int esp_enable = 1; 90 91 static int esp_max_ivlen; /* max iv length over all algorithms */ 92 93 static int esp_input_cb(struct cryptop *op); 94 static int esp_output_cb(struct cryptop *crp); 95 96 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT }; 97 98 static pool_cache_t esp_tdb_crypto_pool_cache; 99 static size_t esp_pool_item_size; 100 101 /* 102 * NB: this is public for use by the PF_KEY support. 103 * NB: if you add support here; be sure to add code to esp_attach below! 104 */ 105 const struct enc_xform * 106 esp_algorithm_lookup(int alg) 107 { 108 109 switch (alg) { 110 case SADB_EALG_DESCBC: 111 return &enc_xform_des; 112 case SADB_EALG_3DESCBC: 113 return &enc_xform_3des; 114 case SADB_X_EALG_AES: 115 return &enc_xform_aes; 116 case SADB_X_EALG_BLOWFISHCBC: 117 return &enc_xform_blf; 118 case SADB_X_EALG_CAST128CBC: 119 return &enc_xform_cast5; 120 case SADB_X_EALG_SKIPJACK: 121 return &enc_xform_skipjack; 122 case SADB_X_EALG_CAMELLIACBC: 123 return &enc_xform_camellia; 124 case SADB_X_EALG_AESCTR: 125 return &enc_xform_aes_ctr; 126 case SADB_X_EALG_AESGCM16: 127 return &enc_xform_aes_gcm; 128 case SADB_X_EALG_AESGMAC: 129 return &enc_xform_aes_gmac; 130 case SADB_EALG_NULL: 131 return &enc_xform_null; 132 } 133 return NULL; 134 } 135 136 size_t 137 esp_hdrsiz(const struct secasvar *sav) 138 { 139 size_t size; 140 141 if (sav != NULL) { 142 /*XXX not right for null algorithm--does it matter??*/ 143 KASSERT(sav->tdb_encalgxform != NULL); 144 145 /* 146 * base header size 147 * + iv length for CBC mode 148 * + max pad length 149 * + sizeof(esp trailer) 150 * + icv length (if any). 151 */ 152 if (sav->flags & SADB_X_EXT_OLD) 153 size = sizeof(struct esp); 154 else 155 size = sizeof(struct newesp); 156 size += sav->tdb_encalgxform->ivsize + 9 + 157 sizeof(struct esptail); 158 159 /*XXX need alg check???*/ 160 if (sav->tdb_authalgxform != NULL && sav->replay) 161 size += ah_authsiz(sav); 162 } else { 163 /* 164 * base header size 165 * + max iv length for CBC mode 166 * + max pad length 167 * + sizeof(esp trailer) 168 * + max icv supported. 169 */ 170 size = sizeof(struct newesp) + esp_max_ivlen + 9 + 171 sizeof(struct esptail) + ah_authsiz(NULL); 172 } 173 return size; 174 } 175 176 /* 177 * esp_init() is called when an SPI is being set up. 178 */ 179 static int 180 esp_init(struct secasvar *sav, const struct xformsw *xsp) 181 { 182 const struct enc_xform *txform; 183 struct cryptoini cria, crie, *cr; 184 int keylen; 185 int error; 186 187 txform = esp_algorithm_lookup(sav->alg_enc); 188 if (txform == NULL) { 189 DPRINTF("unsupported encryption algorithm %d\n", 190 sav->alg_enc); 191 return EINVAL; 192 } 193 if (sav->key_enc == NULL) { 194 DPRINTF("no encoding key for %s algorithm\n", 195 txform->name); 196 return EINVAL; 197 } 198 if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) { 199 DPRINTF("4-byte IV not supported with protocol\n"); 200 return EINVAL; 201 } 202 keylen = _KEYLEN(sav->key_enc); 203 if (txform->minkey > keylen || keylen > txform->maxkey) { 204 DPRINTF("invalid key length %u, must be in " 205 "the range [%u..%u] for algorithm %s\n", 206 keylen, txform->minkey, txform->maxkey, txform->name); 207 return EINVAL; 208 } 209 210 sav->ivlen = txform->ivsize; 211 212 /* 213 * Setup AH-related state. 214 */ 215 if (sav->alg_auth != 0) { 216 error = ah_init0(sav, xsp, &cria); 217 if (error) 218 return error; 219 } 220 221 /* NB: override anything set in ah_init0 */ 222 sav->tdb_xform = xsp; 223 sav->tdb_encalgxform = txform; 224 225 switch (sav->alg_enc) { 226 case SADB_X_EALG_AESGCM16: 227 case SADB_X_EALG_AESGMAC: 228 switch (keylen) { 229 case 20: 230 sav->alg_auth = SADB_X_AALG_AES128GMAC; 231 sav->tdb_authalgxform = &auth_hash_gmac_aes_128; 232 break; 233 case 28: 234 sav->alg_auth = SADB_X_AALG_AES192GMAC; 235 sav->tdb_authalgxform = &auth_hash_gmac_aes_192; 236 break; 237 case 36: 238 sav->alg_auth = SADB_X_AALG_AES256GMAC; 239 sav->tdb_authalgxform = &auth_hash_gmac_aes_256; 240 break; 241 default: 242 DPRINTF("invalid key length %u, must be either of " 243 "20, 28 or 36\n", keylen); 244 return EINVAL; 245 } 246 247 memset(&cria, 0, sizeof(cria)); 248 cria.cri_alg = sav->tdb_authalgxform->type; 249 cria.cri_klen = _KEYBITS(sav->key_enc); 250 cria.cri_key = _KEYBUF(sav->key_enc); 251 break; 252 default: 253 break; 254 } 255 256 /* Initialize crypto session. */ 257 memset(&crie, 0, sizeof(crie)); 258 crie.cri_alg = sav->tdb_encalgxform->type; 259 crie.cri_klen = _KEYBITS(sav->key_enc); 260 crie.cri_key = _KEYBUF(sav->key_enc); 261 /* XXX Rounds ? */ 262 263 if (sav->tdb_authalgxform && sav->tdb_encalgxform) { 264 /* init both auth & enc */ 265 crie.cri_next = &cria; 266 cr = &crie; 267 } else if (sav->tdb_encalgxform) { 268 cr = &crie; 269 } else if (sav->tdb_authalgxform) { 270 cr = &cria; 271 } else { 272 /* XXX cannot happen? */ 273 DPRINTF("no encoding OR authentication xform!\n"); 274 return EINVAL; 275 } 276 277 return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support); 278 } 279 280 /* 281 * Paranoia. 282 */ 283 static int 284 esp_zeroize(struct secasvar *sav) 285 { 286 /* NB: ah_zerorize free's the crypto session state */ 287 int error = ah_zeroize(sav); 288 289 if (sav->key_enc) { 290 explicit_memset(_KEYBUF(sav->key_enc), 0, 291 _KEYLEN(sav->key_enc)); 292 } 293 sav->tdb_encalgxform = NULL; 294 sav->tdb_xform = NULL; 295 return error; 296 } 297 298 /* 299 * ESP input processing, called (eventually) through the protocol switch. 300 */ 301 static int 302 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) 303 { 304 const struct auth_hash *esph; 305 const struct enc_xform *espx; 306 struct tdb_crypto *tc; 307 int plen, alen, hlen, error, stat = ESP_STAT_CRYPTO; 308 struct newesp *esp; 309 struct cryptodesc *crde; 310 struct cryptop *crp; 311 312 KASSERT(sav != NULL); 313 KASSERT(sav->tdb_encalgxform != NULL); 314 KASSERTMSG((skip & 3) == 0 && (m->m_pkthdr.len & 3) == 0, 315 "misaligned packet, skip %u pkt len %u", 316 skip, m->m_pkthdr.len); 317 318 /* XXX don't pullup, just copy header */ 319 M_REGION_GET(esp, struct newesp *, m, skip, sizeof(struct newesp)); 320 if (esp == NULL) { 321 /* m already freed */ 322 return ENOBUFS; 323 } 324 325 esph = sav->tdb_authalgxform; 326 espx = sav->tdb_encalgxform; 327 KASSERT(espx != NULL); 328 329 /* Determine the ESP header length */ 330 if (sav->flags & SADB_X_EXT_OLD) 331 hlen = sizeof(struct esp) + sav->ivlen; 332 else 333 hlen = sizeof(struct newesp) + sav->ivlen; 334 /* Authenticator hash size */ 335 alen = esph ? esph->authsize : 0; 336 337 /* 338 * Verify payload length is multiple of encryption algorithm block 339 * size. 340 * 341 * The payload must also be 4-byte-aligned. This is implicitly 342 * verified here too, since the blocksize is always 4-byte-aligned. 343 */ 344 plen = m->m_pkthdr.len - (skip + hlen + alen); 345 KASSERT((espx->blocksize & 3) == 0); 346 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) { 347 char buf[IPSEC_ADDRSTRLEN]; 348 DPRINTF("payload of %d octets not a multiple of %d octets," 349 " SA %s/%08lx\n", plen, espx->blocksize, 350 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 351 (u_long) ntohl(sav->spi)); 352 stat = ESP_STAT_BADILEN; 353 error = EINVAL; 354 goto out; 355 } 356 357 /* 358 * Check sequence number. 359 */ 360 if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) { 361 char logbuf[IPSEC_LOGSASTRLEN]; 362 DPRINTF("packet replay check for %s\n", 363 ipsec_logsastr(sav, logbuf, sizeof(logbuf))); 364 stat = ESP_STAT_REPLAY; 365 error = EACCES; 366 goto out; 367 } 368 369 /* Update the counters */ 370 ESP_STATADD(ESP_STAT_IBYTES, plen); 371 372 /* Get crypto descriptors */ 373 crp = crypto_getreq(esph ? 2 : 1); 374 if (crp == NULL) { 375 DPRINTF("failed to acquire crypto descriptors\n"); 376 error = ENOBUFS; 377 goto out; 378 } 379 380 /* Get IPsec-specific opaque pointer */ 381 size_t extra __diagused = esph == NULL ? 0 : alen; 382 KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size, 383 "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n", 384 sizeof(*tc) + extra, esp_pool_item_size); 385 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 386 if (tc == NULL) { 387 DPRINTF("failed to allocate tdb_crypto\n"); 388 error = ENOBUFS; 389 goto out1; 390 } 391 392 error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT); 393 if (error) { 394 DPRINTF("m_makewritable failed\n"); 395 goto out2; 396 } 397 398 if (esph) { 399 struct cryptodesc *crda; 400 401 KASSERT(crp->crp_desc != NULL); 402 crda = crp->crp_desc; 403 404 /* Authentication descriptor */ 405 crda->crd_skip = skip; 406 if (espx->type == CRYPTO_AES_GCM_16) 407 crda->crd_len = hlen - sav->ivlen; 408 else 409 crda->crd_len = m->m_pkthdr.len - (skip + alen); 410 crda->crd_inject = m->m_pkthdr.len - alen; 411 412 crda->crd_alg = esph->type; 413 if (espx->type == CRYPTO_AES_GCM_16 || 414 espx->type == CRYPTO_AES_GMAC) { 415 crda->crd_key = _KEYBUF(sav->key_enc); 416 crda->crd_klen = _KEYBITS(sav->key_enc); 417 } else { 418 crda->crd_key = _KEYBUF(sav->key_auth); 419 crda->crd_klen = _KEYBITS(sav->key_auth); 420 } 421 422 /* Copy the authenticator */ 423 m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1)); 424 425 /* Chain authentication request */ 426 crde = crda->crd_next; 427 } else { 428 crde = crp->crp_desc; 429 } 430 431 { 432 int s = pserialize_read_enter(); 433 434 /* 435 * Take another reference to the SA for opencrypto callback. 436 */ 437 if (__predict_false(sav->state == SADB_SASTATE_DEAD)) { 438 pserialize_read_exit(s); 439 stat = ESP_STAT_NOTDB; 440 error = ENOENT; 441 goto out2; 442 } 443 KEY_SA_REF(sav); 444 pserialize_read_exit(s); 445 } 446 447 /* Crypto operation descriptor */ 448 crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ 449 crp->crp_flags = CRYPTO_F_IMBUF; 450 crp->crp_buf = m; 451 crp->crp_callback = esp_input_cb; 452 crp->crp_sid = sav->tdb_cryptoid; 453 crp->crp_opaque = tc; 454 455 /* These are passed as-is to the callback */ 456 tc->tc_spi = sav->spi; 457 tc->tc_dst = sav->sah->saidx.dst; 458 tc->tc_proto = sav->sah->saidx.proto; 459 tc->tc_protoff = protoff; 460 tc->tc_skip = skip; 461 tc->tc_sav = sav; 462 463 /* Decryption descriptor */ 464 KASSERTMSG(crde != NULL, "null esp crypto descriptor"); 465 crde->crd_skip = skip + hlen; 466 if (espx->type == CRYPTO_AES_GMAC) 467 crde->crd_len = 0; 468 else 469 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 470 crde->crd_inject = skip + hlen - sav->ivlen; 471 crde->crd_alg = espx->type; 472 crde->crd_key = _KEYBUF(sav->key_enc); 473 crde->crd_klen = _KEYBITS(sav->key_enc); 474 /* XXX Rounds ? */ 475 476 return crypto_dispatch(crp); 477 478 out2: 479 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 480 out1: 481 crypto_freereq(crp); 482 out: 483 ESP_STATINC(stat); 484 m_freem(m); 485 return error; 486 } 487 488 #ifdef INET6 489 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do { \ 490 if (saidx->dst.sa.sa_family == AF_INET6) { \ 491 error = ipsec6_common_input_cb(m, sav, skip, protoff); \ 492 } else { \ 493 error = ipsec4_common_input_cb(m, sav, skip, protoff); \ 494 } \ 495 } while (0) 496 #else 497 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) \ 498 (error = ipsec4_common_input_cb(m, sav, skip, protoff)) 499 #endif 500 501 /* 502 * ESP input callback from the crypto driver. 503 */ 504 static int 505 esp_input_cb(struct cryptop *crp) 506 { 507 char buf[IPSEC_ADDRSTRLEN]; 508 uint8_t lastthree[3], aalg[AH_ALEN_MAX]; 509 int hlen, skip, protoff, error; 510 struct mbuf *m; 511 const struct auth_hash *esph; 512 struct tdb_crypto *tc; 513 struct secasvar *sav; 514 struct secasindex *saidx; 515 void *ptr; 516 IPSEC_DECLARE_LOCK_VARIABLE; 517 518 KASSERT(crp->crp_desc != NULL); 519 KASSERT(crp->crp_opaque != NULL); 520 521 tc = crp->crp_opaque; 522 skip = tc->tc_skip; 523 protoff = tc->tc_protoff; 524 m = crp->crp_buf; 525 526 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 527 528 sav = tc->tc_sav; 529 saidx = &sav->sah->saidx; 530 KASSERTMSG(saidx->dst.sa.sa_family == AF_INET || 531 saidx->dst.sa.sa_family == AF_INET6, 532 "unexpected protocol family %u", saidx->dst.sa.sa_family); 533 534 esph = sav->tdb_authalgxform; 535 536 /* Check for crypto errors */ 537 if (crp->crp_etype) { 538 /* Reset the session ID */ 539 if (sav->tdb_cryptoid != 0) 540 sav->tdb_cryptoid = crp->crp_sid; 541 542 if (crp->crp_etype == EAGAIN) { 543 KEY_SA_UNREF(&sav); 544 IPSEC_RELEASE_GLOBAL_LOCKS(); 545 return crypto_dispatch(crp); 546 } 547 548 ESP_STATINC(ESP_STAT_NOXFORM); 549 DPRINTF("crypto error %d\n", crp->crp_etype); 550 error = crp->crp_etype; 551 goto bad; 552 } 553 554 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 555 556 /* If authentication was performed, check now. */ 557 if (esph != NULL) { 558 /* 559 * If we have a tag, it means an IPsec-aware NIC did 560 * the verification for us. Otherwise we need to 561 * check the authentication calculation. 562 */ 563 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 564 /* Copy the authenticator from the packet */ 565 m_copydata(m, m->m_pkthdr.len - esph->authsize, 566 esph->authsize, aalg); 567 568 ptr = (tc + 1); 569 570 /* Verify authenticator */ 571 if (!consttime_memequal(ptr, aalg, esph->authsize)) { 572 DPRINTF("authentication hash mismatch " 573 "for packet in SA %s/%08lx\n", 574 ipsec_address(&saidx->dst, buf, 575 sizeof(buf)), (u_long) ntohl(sav->spi)); 576 ESP_STATINC(ESP_STAT_BADAUTH); 577 error = EACCES; 578 goto bad; 579 } 580 581 /* Remove trailing authenticator */ 582 m_adj(m, -(esph->authsize)); 583 } 584 585 /* Release the crypto descriptors */ 586 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 587 tc = NULL; 588 crypto_freereq(crp); 589 crp = NULL; 590 591 /* 592 * Packet is now decrypted. 593 */ 594 m->m_flags |= M_DECRYPTED; 595 596 /* 597 * Update replay sequence number, if appropriate. 598 */ 599 if (sav->replay) { 600 uint32_t seq; 601 602 m_copydata(m, skip + offsetof(struct newesp, esp_seq), 603 sizeof(seq), &seq); 604 if (ipsec_updatereplay(ntohl(seq), sav)) { 605 char logbuf[IPSEC_LOGSASTRLEN]; 606 DPRINTF("packet replay check for %s\n", 607 ipsec_logsastr(sav, logbuf, sizeof(logbuf))); 608 ESP_STATINC(ESP_STAT_REPLAY); 609 error = EACCES; 610 goto bad; 611 } 612 } 613 614 /* Determine the ESP header length */ 615 if (sav->flags & SADB_X_EXT_OLD) 616 hlen = sizeof(struct esp) + sav->ivlen; 617 else 618 hlen = sizeof(struct newesp) + sav->ivlen; 619 620 /* Remove the ESP header and IV from the mbuf. */ 621 error = m_striphdr(m, skip, hlen); 622 if (error) { 623 ESP_STATINC(ESP_STAT_HDROPS); 624 DPRINTF("bad mbuf chain, SA %s/%08lx\n", 625 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 626 (u_long) ntohl(sav->spi)); 627 goto bad; 628 } 629 630 /* Save the last three bytes of decrypted data */ 631 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree); 632 633 /* Verify pad length */ 634 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) { 635 ESP_STATINC(ESP_STAT_BADILEN); 636 DPRINTF("invalid padding length %d " 637 "for %u byte packet in SA %s/%08lx\n", 638 lastthree[1], m->m_pkthdr.len - skip, 639 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 640 (u_long) ntohl(sav->spi)); 641 error = EINVAL; 642 goto bad; 643 } 644 645 /* Verify correct decryption by checking the last padding bytes */ 646 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) { 647 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) { 648 ESP_STATINC(ESP_STAT_BADENC); 649 DPRINTF("decryption failed for packet in SA " 650 "%s/%08lx\n", 651 ipsec_address(&sav->sah->saidx.dst, buf, 652 sizeof(buf)), (u_long) ntohl(sav->spi)); 653 DPRINTF("%x %x\n", lastthree[0], 654 lastthree[1]); 655 error = EINVAL; 656 goto bad; 657 } 658 } 659 660 /* Trim the mbuf chain to remove trailing authenticator and padding */ 661 m_adj(m, -(lastthree[1] + 2)); 662 663 /* Restore the Next Protocol field */ 664 m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2); 665 666 IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff); 667 668 KEY_SA_UNREF(&sav); 669 IPSEC_RELEASE_GLOBAL_LOCKS(); 670 return error; 671 bad: 672 if (sav) 673 KEY_SA_UNREF(&sav); 674 IPSEC_RELEASE_GLOBAL_LOCKS(); 675 if (m != NULL) 676 m_freem(m); 677 if (tc != NULL) 678 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 679 if (crp != NULL) 680 crypto_freereq(crp); 681 return error; 682 } 683 684 /* 685 * ESP output routine, called by ipsec[46]_process_packet(). 686 */ 687 static int 688 esp_output(struct mbuf *m, const struct ipsecrequest *isr, struct secasvar *sav, 689 int skip, int protoff, int flags) 690 { 691 char buf[IPSEC_ADDRSTRLEN]; 692 const struct enc_xform *espx; 693 const struct auth_hash *esph; 694 int hlen, rlen, tlen, padlen, blks, alen, i, roff; 695 struct mbuf *mo = NULL; 696 struct tdb_crypto *tc; 697 struct secasindex *saidx; 698 unsigned char *tail; 699 uint8_t prot; 700 int error, maxpacketsize; 701 struct esptail *esptail; 702 struct cryptodesc *crde, *crda; 703 struct cryptop *crp; 704 705 esph = sav->tdb_authalgxform; 706 espx = sav->tdb_encalgxform; 707 KASSERT(espx != NULL); 708 709 /* Determine the ESP header length */ 710 if (sav->flags & SADB_X_EXT_OLD) 711 hlen = sizeof(struct esp) + sav->ivlen; 712 else 713 hlen = sizeof(struct newesp) + sav->ivlen; 714 /* Authenticator hash size */ 715 alen = esph ? esph->authsize : 0; 716 717 /* 718 * NB: The null encoding transform has a blocksize of 4 719 * so that headers are properly aligned. 720 */ 721 blks = espx->blocksize; /* IV blocksize */ 722 723 /* Raw payload length. */ 724 rlen = m->m_pkthdr.len - skip; 725 726 /* Encryption padding. */ 727 padlen = ((blks - ((rlen + sizeof(struct esptail)) % blks)) % blks); 728 729 /* Length of what we append (tail). */ 730 tlen = padlen + sizeof(struct esptail) + alen; 731 732 ESP_STATINC(ESP_STAT_OUTPUT); 733 734 saidx = &sav->sah->saidx; 735 /* Check for maximum packet size violations. */ 736 switch (saidx->dst.sa.sa_family) { 737 #ifdef INET 738 case AF_INET: 739 maxpacketsize = IP_MAXPACKET; 740 break; 741 #endif 742 #ifdef INET6 743 case AF_INET6: 744 maxpacketsize = IPV6_MAXPACKET; 745 break; 746 #endif 747 default: 748 DPRINTF("unknown/unsupported protocol family %d, " 749 "SA %s/%08lx\n", saidx->dst.sa.sa_family, 750 ipsec_address(&saidx->dst, buf, sizeof(buf)), 751 (u_long)ntohl(sav->spi)); 752 ESP_STATINC(ESP_STAT_NOPF); 753 error = EPFNOSUPPORT; 754 goto bad; 755 } 756 if (skip + hlen + rlen + tlen > maxpacketsize) { 757 DPRINTF("packet in SA %s/%08lx got too big (len %u, " 758 "max len %u)\n", 759 ipsec_address(&saidx->dst, buf, sizeof(buf)), 760 (u_long) ntohl(sav->spi), 761 skip + hlen + rlen + tlen, maxpacketsize); 762 ESP_STATINC(ESP_STAT_TOOBIG); 763 error = EMSGSIZE; 764 goto bad; 765 } 766 767 /* Update the counters. */ 768 ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip); 769 770 m = m_clone(m); 771 if (m == NULL) { 772 DPRINTF("cannot clone mbuf chain, SA %s/%08lx\n", 773 ipsec_address(&saidx->dst, buf, sizeof(buf)), 774 (u_long) ntohl(sav->spi)); 775 ESP_STATINC(ESP_STAT_HDROPS); 776 error = ENOBUFS; 777 goto bad; 778 } 779 780 /* Inject ESP header. */ 781 mo = m_makespace(m, skip, hlen, &roff); 782 if (mo == NULL) { 783 DPRINTF("failed to inject %u byte ESP hdr for SA " 784 "%s/%08lx\n", hlen, 785 ipsec_address(&saidx->dst, buf, sizeof(buf)), 786 (u_long) ntohl(sav->spi)); 787 ESP_STATINC(ESP_STAT_HDROPS); 788 error = ENOBUFS; 789 goto bad; 790 } 791 792 /* Initialize ESP header. */ 793 memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t)); 794 if (sav->replay) { 795 uint32_t replay; 796 797 #ifdef IPSEC_DEBUG 798 /* Emulate replay attack when ipsec_replay is TRUE. */ 799 if (!ipsec_replay) 800 #endif 801 sav->replay->count++; 802 803 replay = htonl(sav->replay->count); 804 memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay, 805 sizeof(uint32_t)); 806 } 807 808 /* 809 * Grow the mbuf, we will append data at the tail. 810 */ 811 tail = m_pad(m, tlen); 812 if (tail == NULL) { 813 DPRINTF("m_pad failed for SA %s/%08lx\n", 814 ipsec_address(&saidx->dst, buf, sizeof(buf)), 815 (u_long) ntohl(sav->spi)); 816 m = NULL; 817 error = ENOBUFS; 818 goto bad; 819 } 820 821 /* 822 * Add padding: random, zero, or self-describing. 823 */ 824 switch (sav->flags & SADB_X_EXT_PMASK) { 825 case SADB_X_EXT_PSEQ: 826 for (i = 0; i < padlen; i++) 827 tail[i] = i + 1; 828 break; 829 case SADB_X_EXT_PRAND: 830 (void)cprng_fast(tail, padlen); 831 break; 832 case SADB_X_EXT_PZERO: 833 default: 834 memset(tail, 0, padlen); 835 break; 836 } 837 838 /* Build the ESP Trailer. */ 839 esptail = (struct esptail *)&tail[padlen]; 840 esptail->esp_padlen = padlen; 841 m_copydata(m, protoff, sizeof(uint8_t), &esptail->esp_nxt); 842 843 /* Fix Next Protocol in IPv4/IPv6 header. */ 844 prot = IPPROTO_ESP; 845 m_copyback(m, protoff, sizeof(uint8_t), &prot); 846 847 /* Get crypto descriptors. */ 848 crp = crypto_getreq(esph ? 2 : 1); 849 if (crp == NULL) { 850 DPRINTF("failed to acquire crypto descriptors\n"); 851 ESP_STATINC(ESP_STAT_CRYPTO); 852 error = ENOBUFS; 853 goto bad; 854 } 855 856 /* Get the descriptors. */ 857 crde = crp->crp_desc; 858 crda = crde->crd_next; 859 860 /* Encryption descriptor. */ 861 crde->crd_skip = skip + hlen; 862 if (espx->type == CRYPTO_AES_GMAC) 863 crde->crd_len = 0; 864 else 865 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 866 crde->crd_flags = CRD_F_ENCRYPT; 867 crde->crd_inject = skip + hlen - sav->ivlen; 868 crde->crd_alg = espx->type; 869 crde->crd_key = _KEYBUF(sav->key_enc); 870 crde->crd_klen = _KEYBITS(sav->key_enc); 871 /* XXX Rounds ? */ 872 873 /* IPsec-specific opaque crypto info. */ 874 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 875 if (tc == NULL) { 876 crypto_freereq(crp); 877 DPRINTF("failed to allocate tdb_crypto\n"); 878 ESP_STATINC(ESP_STAT_CRYPTO); 879 error = ENOBUFS; 880 goto bad; 881 } 882 883 { 884 int s = pserialize_read_enter(); 885 886 /* 887 * Take another reference to the SP and the SA for opencrypto callback. 888 */ 889 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD || 890 sav->state == SADB_SASTATE_DEAD)) { 891 pserialize_read_exit(s); 892 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 893 crypto_freereq(crp); 894 ESP_STATINC(ESP_STAT_NOTDB); 895 error = ENOENT; 896 goto bad; 897 } 898 KEY_SP_REF(isr->sp); 899 KEY_SA_REF(sav); 900 pserialize_read_exit(s); 901 } 902 903 /* Callback parameters */ 904 tc->tc_isr = isr; 905 tc->tc_spi = sav->spi; 906 tc->tc_dst = saidx->dst; 907 tc->tc_proto = saidx->proto; 908 tc->tc_flags = flags; 909 tc->tc_sav = sav; 910 911 /* Crypto operation descriptor. */ 912 crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ 913 crp->crp_flags = CRYPTO_F_IMBUF; 914 crp->crp_buf = m; 915 crp->crp_callback = esp_output_cb; 916 crp->crp_opaque = tc; 917 crp->crp_sid = sav->tdb_cryptoid; 918 919 if (esph) { 920 /* Authentication descriptor. */ 921 crda->crd_skip = skip; 922 if (espx->type == CRYPTO_AES_GCM_16) 923 crda->crd_len = hlen - sav->ivlen; 924 else 925 crda->crd_len = m->m_pkthdr.len - (skip + alen); 926 crda->crd_inject = m->m_pkthdr.len - alen; 927 928 /* Authentication operation. */ 929 crda->crd_alg = esph->type; 930 if (espx->type == CRYPTO_AES_GCM_16 || 931 espx->type == CRYPTO_AES_GMAC) { 932 crda->crd_key = _KEYBUF(sav->key_enc); 933 crda->crd_klen = _KEYBITS(sav->key_enc); 934 } else { 935 crda->crd_key = _KEYBUF(sav->key_auth); 936 crda->crd_klen = _KEYBITS(sav->key_auth); 937 } 938 } 939 940 return crypto_dispatch(crp); 941 942 bad: 943 if (m) 944 m_freem(m); 945 return error; 946 } 947 948 /* 949 * ESP output callback from the crypto driver. 950 */ 951 static int 952 esp_output_cb(struct cryptop *crp) 953 { 954 struct tdb_crypto *tc; 955 const struct ipsecrequest *isr; 956 struct secasvar *sav; 957 struct mbuf *m; 958 int err, error, flags; 959 IPSEC_DECLARE_LOCK_VARIABLE; 960 961 KASSERT(crp->crp_opaque != NULL); 962 tc = crp->crp_opaque; 963 m = crp->crp_buf; 964 965 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 966 967 isr = tc->tc_isr; 968 sav = tc->tc_sav; 969 970 /* Check for crypto errors. */ 971 if (crp->crp_etype) { 972 /* Reset session ID. */ 973 if (sav->tdb_cryptoid != 0) 974 sav->tdb_cryptoid = crp->crp_sid; 975 976 if (crp->crp_etype == EAGAIN) { 977 IPSEC_RELEASE_GLOBAL_LOCKS(); 978 return crypto_dispatch(crp); 979 } 980 981 ESP_STATINC(ESP_STAT_NOXFORM); 982 DPRINTF("crypto error %d\n", crp->crp_etype); 983 error = crp->crp_etype; 984 goto bad; 985 } 986 987 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 988 if (sav->tdb_authalgxform != NULL) 989 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 990 991 flags = tc->tc_flags; 992 /* Release crypto descriptors. */ 993 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 994 crypto_freereq(crp); 995 996 #ifdef IPSEC_DEBUG 997 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */ 998 if (ipsec_integrity) { 999 static unsigned char ipseczeroes[AH_ALEN_MAX]; 1000 const struct auth_hash *esph; 1001 1002 /* 1003 * Corrupt HMAC if we want to test integrity verification of 1004 * the other side. 1005 */ 1006 esph = sav->tdb_authalgxform; 1007 if (esph != NULL) { 1008 m_copyback(m, m->m_pkthdr.len - esph->authsize, 1009 esph->authsize, ipseczeroes); 1010 } 1011 } 1012 #endif 1013 1014 /* NB: m is reclaimed by ipsec_process_done. */ 1015 err = ipsec_process_done(m, isr, sav, flags); 1016 KEY_SA_UNREF(&sav); 1017 KEY_SP_UNREF(&isr->sp); 1018 IPSEC_RELEASE_GLOBAL_LOCKS(); 1019 return err; 1020 1021 bad: 1022 if (sav) 1023 KEY_SA_UNREF(&sav); 1024 KEY_SP_UNREF(&isr->sp); 1025 IPSEC_RELEASE_GLOBAL_LOCKS(); 1026 if (m) 1027 m_freem(m); 1028 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 1029 crypto_freereq(crp); 1030 return error; 1031 } 1032 1033 static struct xformsw esp_xformsw = { 1034 .xf_type = XF_ESP, 1035 .xf_flags = XFT_CONF|XFT_AUTH, 1036 .xf_name = "IPsec ESP", 1037 .xf_init = esp_init, 1038 .xf_zeroize = esp_zeroize, 1039 .xf_input = esp_input, 1040 .xf_output = esp_output, 1041 .xf_next = NULL, 1042 }; 1043 1044 void 1045 esp_attach(void) 1046 { 1047 1048 espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS); 1049 1050 extern int ah_max_authsize; 1051 KASSERT(ah_max_authsize != 0); 1052 esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize; 1053 esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size, 1054 coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET, 1055 NULL, NULL, NULL); 1056 1057 #define MAXIV(xform) \ 1058 if (xform.ivsize > esp_max_ivlen) \ 1059 esp_max_ivlen = xform.ivsize \ 1060 1061 esp_max_ivlen = 0; 1062 MAXIV(enc_xform_des); /* SADB_EALG_DESCBC */ 1063 MAXIV(enc_xform_3des); /* SADB_EALG_3DESCBC */ 1064 MAXIV(enc_xform_aes); /* SADB_X_EALG_AES */ 1065 MAXIV(enc_xform_blf); /* SADB_X_EALG_BLOWFISHCBC */ 1066 MAXIV(enc_xform_cast5); /* SADB_X_EALG_CAST128CBC */ 1067 MAXIV(enc_xform_skipjack); /* SADB_X_EALG_SKIPJACK */ 1068 MAXIV(enc_xform_camellia); /* SADB_X_EALG_CAMELLIACBC */ 1069 MAXIV(enc_xform_aes_ctr); /* SADB_X_EALG_AESCTR */ 1070 MAXIV(enc_xform_null); /* SADB_EALG_NULL */ 1071 1072 xform_register(&esp_xformsw); 1073 #undef MAXIV 1074 } 1075