1 /* $NetBSD: xform_esp.c,v 1.97 2019/01/27 02:08:48 pgoyette Exp $ */ 2 /* $FreeBSD: xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */ 3 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ 4 5 /* 6 * The authors of this code are John Ioannidis (ji@tla.org), 7 * Angelos D. Keromytis (kermit@csd.uch.gr) and 8 * Niels Provos (provos@physnet.uni-hamburg.de). 9 * 10 * The original version of this code was written by John Ioannidis 11 * for BSD/OS in Athens, Greece, in November 1995. 12 * 13 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, 14 * by Angelos D. Keromytis. 15 * 16 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis 17 * and Niels Provos. 18 * 19 * Additional features in 1999 by Angelos D. Keromytis. 20 * 21 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, 22 * Angelos D. Keromytis and Niels Provos. 23 * Copyright (c) 2001 Angelos D. Keromytis. 24 * 25 * Permission to use, copy, and modify this software with or without fee 26 * is hereby granted, provided that this entire notice is included in 27 * all copies of any software which is or includes a copy or 28 * modification of this software. 29 * You may use this code under the GNU public license if you so wish. Please 30 * contribute changes back to the authors under this freer than GPL license 31 * so that we may further the use of strong encryption without limitations to 32 * all. 33 * 34 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 35 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 36 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 37 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 38 * PURPOSE. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.97 2019/01/27 02:08:48 pgoyette Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_inet.h" 46 #include "opt_ipsec.h" 47 #endif 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/mbuf.h> 52 #include <sys/socket.h> 53 #include <sys/syslog.h> 54 #include <sys/kernel.h> 55 #include <sys/sysctl.h> 56 #include <sys/cprng.h> 57 #include <sys/pool.h> 58 #include <sys/pserialize.h> 59 60 #include <net/if.h> 61 62 #include <netinet/in.h> 63 #include <netinet/in_systm.h> 64 #include <netinet/ip.h> 65 #include <netinet/ip_ecn.h> 66 #include <netinet/ip6.h> 67 68 #include <net/route.h> 69 #include <netipsec/ipsec.h> 70 #include <netipsec/ipsec_private.h> 71 #include <netipsec/ah.h> 72 #include <netipsec/ah_var.h> 73 #include <netipsec/esp.h> 74 #include <netipsec/esp_var.h> 75 #include <netipsec/xform.h> 76 77 #ifdef INET6 78 #include <netinet6/ip6_var.h> 79 #include <netipsec/ipsec6.h> 80 #endif 81 82 #include <netipsec/key.h> 83 #include <netipsec/key_debug.h> 84 85 #include <opencrypto/cryptodev.h> 86 87 percpu_t *espstat_percpu; 88 89 int esp_enable = 1; 90 91 static int esp_max_ivlen; /* max iv length over all algorithms */ 92 93 static int esp_input_cb(struct cryptop *op); 94 static int esp_output_cb(struct cryptop *crp); 95 96 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT }; 97 98 static pool_cache_t esp_tdb_crypto_pool_cache; 99 static size_t esp_pool_item_size; 100 101 /* 102 * NB: this is public for use by the PF_KEY support. 103 * NB: if you add support here; be sure to add code to esp_attach below! 104 */ 105 const struct enc_xform * 106 esp_algorithm_lookup(int alg) 107 { 108 109 switch (alg) { 110 case SADB_EALG_DESCBC: 111 return &enc_xform_des; 112 case SADB_EALG_3DESCBC: 113 return &enc_xform_3des; 114 case SADB_X_EALG_AES: 115 return &enc_xform_rijndael128; 116 case SADB_X_EALG_BLOWFISHCBC: 117 return &enc_xform_blf; 118 case SADB_X_EALG_CAST128CBC: 119 return &enc_xform_cast5; 120 case SADB_X_EALG_SKIPJACK: 121 return &enc_xform_skipjack; 122 case SADB_X_EALG_CAMELLIACBC: 123 return &enc_xform_camellia; 124 case SADB_X_EALG_AESCTR: 125 return &enc_xform_aes_ctr; 126 case SADB_X_EALG_AESGCM16: 127 return &enc_xform_aes_gcm; 128 case SADB_X_EALG_AESGMAC: 129 return &enc_xform_aes_gmac; 130 case SADB_EALG_NULL: 131 return &enc_xform_null; 132 } 133 return NULL; 134 } 135 136 size_t 137 esp_hdrsiz(const struct secasvar *sav) 138 { 139 size_t size; 140 141 if (sav != NULL) { 142 /*XXX not right for null algorithm--does it matter??*/ 143 KASSERT(sav->tdb_encalgxform != NULL); 144 145 /* 146 * base header size 147 * + iv length for CBC mode 148 * + max pad length 149 * + sizeof(esp trailer) 150 * + icv length (if any). 151 */ 152 if (sav->flags & SADB_X_EXT_OLD) 153 size = sizeof(struct esp); 154 else 155 size = sizeof(struct newesp); 156 size += sav->tdb_encalgxform->ivsize + 9 + 157 sizeof(struct esptail); 158 159 /*XXX need alg check???*/ 160 if (sav->tdb_authalgxform != NULL && sav->replay) 161 size += ah_authsiz(sav); 162 } else { 163 /* 164 * base header size 165 * + max iv length for CBC mode 166 * + max pad length 167 * + sizeof(esp trailer) 168 * + max icv supported. 169 */ 170 size = sizeof(struct newesp) + esp_max_ivlen + 9 + 171 sizeof(struct esptail) + ah_authsiz(NULL); 172 } 173 return size; 174 } 175 176 /* 177 * esp_init() is called when an SPI is being set up. 178 */ 179 static int 180 esp_init(struct secasvar *sav, const struct xformsw *xsp) 181 { 182 const struct enc_xform *txform; 183 struct cryptoini cria, crie, *cr; 184 int keylen; 185 int error; 186 187 txform = esp_algorithm_lookup(sav->alg_enc); 188 if (txform == NULL) { 189 DPRINTF(("%s: unsupported encryption algorithm %d\n", __func__, 190 sav->alg_enc)); 191 return EINVAL; 192 } 193 if (sav->key_enc == NULL) { 194 DPRINTF(("%s: no encoding key for %s algorithm\n", __func__, 195 txform->name)); 196 return EINVAL; 197 } 198 if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) { 199 DPRINTF(("%s: 4-byte IV not supported with protocol\n", 200 __func__)); 201 return EINVAL; 202 } 203 keylen = _KEYLEN(sav->key_enc); 204 if (txform->minkey > keylen || keylen > txform->maxkey) { 205 DPRINTF(("%s: invalid key length %u, must be in " 206 "the range [%u..%u] for algorithm %s\n", __func__, 207 keylen, txform->minkey, txform->maxkey, txform->name)); 208 return EINVAL; 209 } 210 211 sav->ivlen = txform->ivsize; 212 213 /* 214 * Setup AH-related state. 215 */ 216 if (sav->alg_auth != 0) { 217 error = ah_init0(sav, xsp, &cria); 218 if (error) 219 return error; 220 } 221 222 /* NB: override anything set in ah_init0 */ 223 sav->tdb_xform = xsp; 224 sav->tdb_encalgxform = txform; 225 226 switch (sav->alg_enc) { 227 case SADB_X_EALG_AESGCM16: 228 case SADB_X_EALG_AESGMAC: 229 switch (keylen) { 230 case 20: 231 sav->alg_auth = SADB_X_AALG_AES128GMAC; 232 sav->tdb_authalgxform = &auth_hash_gmac_aes_128; 233 break; 234 case 28: 235 sav->alg_auth = SADB_X_AALG_AES192GMAC; 236 sav->tdb_authalgxform = &auth_hash_gmac_aes_192; 237 break; 238 case 36: 239 sav->alg_auth = SADB_X_AALG_AES256GMAC; 240 sav->tdb_authalgxform = &auth_hash_gmac_aes_256; 241 break; 242 default: 243 DPRINTF(("%s: invalid key length %u, must be either of " 244 "20, 28 or 36\n", __func__, keylen)); 245 return EINVAL; 246 } 247 248 memset(&cria, 0, sizeof(cria)); 249 cria.cri_alg = sav->tdb_authalgxform->type; 250 cria.cri_klen = _KEYBITS(sav->key_enc); 251 cria.cri_key = _KEYBUF(sav->key_enc); 252 break; 253 default: 254 break; 255 } 256 257 /* Initialize crypto session. */ 258 memset(&crie, 0, sizeof(crie)); 259 crie.cri_alg = sav->tdb_encalgxform->type; 260 crie.cri_klen = _KEYBITS(sav->key_enc); 261 crie.cri_key = _KEYBUF(sav->key_enc); 262 /* XXX Rounds ? */ 263 264 if (sav->tdb_authalgxform && sav->tdb_encalgxform) { 265 /* init both auth & enc */ 266 crie.cri_next = &cria; 267 cr = &crie; 268 } else if (sav->tdb_encalgxform) { 269 cr = &crie; 270 } else if (sav->tdb_authalgxform) { 271 cr = &cria; 272 } else { 273 /* XXX cannot happen? */ 274 DPRINTF(("%s: no encoding OR authentication xform!\n", 275 __func__)); 276 return EINVAL; 277 } 278 279 return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support); 280 } 281 282 /* 283 * Paranoia. 284 */ 285 static int 286 esp_zeroize(struct secasvar *sav) 287 { 288 /* NB: ah_zerorize free's the crypto session state */ 289 int error = ah_zeroize(sav); 290 291 if (sav->key_enc) { 292 explicit_memset(_KEYBUF(sav->key_enc), 0, 293 _KEYLEN(sav->key_enc)); 294 } 295 sav->tdb_encalgxform = NULL; 296 sav->tdb_xform = NULL; 297 return error; 298 } 299 300 /* 301 * ESP input processing, called (eventually) through the protocol switch. 302 */ 303 static int 304 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) 305 { 306 const struct auth_hash *esph; 307 const struct enc_xform *espx; 308 struct tdb_crypto *tc; 309 int plen, alen, hlen, error, stat = ESP_STAT_CRYPTO; 310 struct newesp *esp; 311 struct cryptodesc *crde; 312 struct cryptop *crp; 313 314 KASSERT(sav != NULL); 315 KASSERT(sav->tdb_encalgxform != NULL); 316 KASSERTMSG((skip & 3) == 0 && (m->m_pkthdr.len & 3) == 0, 317 "misaligned packet, skip %u pkt len %u", 318 skip, m->m_pkthdr.len); 319 320 /* XXX don't pullup, just copy header */ 321 M_REGION_GET(esp, struct newesp *, m, skip, sizeof(struct newesp)); 322 if (esp == NULL) { 323 /* m already freed */ 324 return ENOBUFS; 325 } 326 327 esph = sav->tdb_authalgxform; 328 espx = sav->tdb_encalgxform; 329 KASSERT(espx != NULL); 330 331 /* Determine the ESP header length */ 332 if (sav->flags & SADB_X_EXT_OLD) 333 hlen = sizeof(struct esp) + sav->ivlen; 334 else 335 hlen = sizeof(struct newesp) + sav->ivlen; 336 /* Authenticator hash size */ 337 alen = esph ? esph->authsize : 0; 338 339 /* 340 * Verify payload length is multiple of encryption algorithm block 341 * size. 342 * 343 * The payload must also be 4-byte-aligned. This is implicitly 344 * verified here too, since the blocksize is always 4-byte-aligned. 345 */ 346 plen = m->m_pkthdr.len - (skip + hlen + alen); 347 KASSERT((espx->blocksize & 3) == 0); 348 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) { 349 char buf[IPSEC_ADDRSTRLEN]; 350 DPRINTF(("%s: payload of %d octets not a multiple of %d octets," 351 " SA %s/%08lx\n", __func__, plen, espx->blocksize, 352 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 353 (u_long) ntohl(sav->spi))); 354 stat = ESP_STAT_BADILEN; 355 error = EINVAL; 356 goto out; 357 } 358 359 /* 360 * Check sequence number. 361 */ 362 if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) { 363 char logbuf[IPSEC_LOGSASTRLEN]; 364 DPRINTF(("%s: packet replay check for %s\n", __func__, 365 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); 366 stat = ESP_STAT_REPLAY; 367 error = EACCES; 368 goto out; 369 } 370 371 /* Update the counters */ 372 ESP_STATADD(ESP_STAT_IBYTES, plen); 373 374 /* Get crypto descriptors */ 375 crp = crypto_getreq(esph ? 2 : 1); 376 if (crp == NULL) { 377 DPRINTF(("%s: failed to acquire crypto descriptors\n", 378 __func__)); 379 error = ENOBUFS; 380 goto out; 381 } 382 383 /* Get IPsec-specific opaque pointer */ 384 size_t extra __diagused = esph == NULL ? 0 : alen; 385 KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size, 386 "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n", 387 sizeof(*tc) + extra, esp_pool_item_size); 388 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 389 if (tc == NULL) { 390 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); 391 error = ENOBUFS; 392 goto out1; 393 } 394 395 error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT); 396 if (error) { 397 DPRINTF(("%s: m_makewritable failed\n", __func__)); 398 goto out2; 399 } 400 401 if (esph) { 402 struct cryptodesc *crda; 403 404 KASSERT(crp->crp_desc != NULL); 405 crda = crp->crp_desc; 406 407 /* Authentication descriptor */ 408 crda->crd_skip = skip; 409 if (espx->type == CRYPTO_AES_GCM_16) 410 crda->crd_len = hlen - sav->ivlen; 411 else 412 crda->crd_len = m->m_pkthdr.len - (skip + alen); 413 crda->crd_inject = m->m_pkthdr.len - alen; 414 415 crda->crd_alg = esph->type; 416 if (espx->type == CRYPTO_AES_GCM_16 || 417 espx->type == CRYPTO_AES_GMAC) { 418 crda->crd_key = _KEYBUF(sav->key_enc); 419 crda->crd_klen = _KEYBITS(sav->key_enc); 420 } else { 421 crda->crd_key = _KEYBUF(sav->key_auth); 422 crda->crd_klen = _KEYBITS(sav->key_auth); 423 } 424 425 /* Copy the authenticator */ 426 m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1)); 427 428 /* Chain authentication request */ 429 crde = crda->crd_next; 430 } else { 431 crde = crp->crp_desc; 432 } 433 434 { 435 int s = pserialize_read_enter(); 436 437 /* 438 * Take another reference to the SA for opencrypto callback. 439 */ 440 if (__predict_false(sav->state == SADB_SASTATE_DEAD)) { 441 pserialize_read_exit(s); 442 stat = ESP_STAT_NOTDB; 443 error = ENOENT; 444 goto out2; 445 } 446 KEY_SA_REF(sav); 447 pserialize_read_exit(s); 448 } 449 450 /* Crypto operation descriptor */ 451 crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ 452 crp->crp_flags = CRYPTO_F_IMBUF; 453 crp->crp_buf = m; 454 crp->crp_callback = esp_input_cb; 455 crp->crp_sid = sav->tdb_cryptoid; 456 crp->crp_opaque = tc; 457 458 /* These are passed as-is to the callback */ 459 tc->tc_spi = sav->spi; 460 tc->tc_dst = sav->sah->saidx.dst; 461 tc->tc_proto = sav->sah->saidx.proto; 462 tc->tc_protoff = protoff; 463 tc->tc_skip = skip; 464 tc->tc_sav = sav; 465 466 /* Decryption descriptor */ 467 KASSERTMSG(crde != NULL, "null esp crypto descriptor"); 468 crde->crd_skip = skip + hlen; 469 if (espx->type == CRYPTO_AES_GMAC) 470 crde->crd_len = 0; 471 else 472 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 473 crde->crd_inject = skip + hlen - sav->ivlen; 474 crde->crd_alg = espx->type; 475 crde->crd_key = _KEYBUF(sav->key_enc); 476 crde->crd_klen = _KEYBITS(sav->key_enc); 477 /* XXX Rounds ? */ 478 479 return crypto_dispatch(crp); 480 481 out2: 482 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 483 out1: 484 crypto_freereq(crp); 485 out: 486 ESP_STATINC(stat); 487 m_freem(m); 488 return error; 489 } 490 491 #ifdef INET6 492 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do { \ 493 if (saidx->dst.sa.sa_family == AF_INET6) { \ 494 error = ipsec6_common_input_cb(m, sav, skip, protoff); \ 495 } else { \ 496 error = ipsec4_common_input_cb(m, sav, skip, protoff); \ 497 } \ 498 } while (0) 499 #else 500 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) \ 501 (error = ipsec4_common_input_cb(m, sav, skip, protoff)) 502 #endif 503 504 /* 505 * ESP input callback from the crypto driver. 506 */ 507 static int 508 esp_input_cb(struct cryptop *crp) 509 { 510 char buf[IPSEC_ADDRSTRLEN]; 511 uint8_t lastthree[3], aalg[AH_ALEN_MAX]; 512 int hlen, skip, protoff, error; 513 struct mbuf *m; 514 const struct auth_hash *esph; 515 struct tdb_crypto *tc; 516 struct secasvar *sav; 517 struct secasindex *saidx; 518 void *ptr; 519 IPSEC_DECLARE_LOCK_VARIABLE; 520 521 KASSERT(crp->crp_desc != NULL); 522 KASSERT(crp->crp_opaque != NULL); 523 524 tc = crp->crp_opaque; 525 skip = tc->tc_skip; 526 protoff = tc->tc_protoff; 527 m = crp->crp_buf; 528 529 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 530 531 sav = tc->tc_sav; 532 saidx = &sav->sah->saidx; 533 KASSERTMSG(saidx->dst.sa.sa_family == AF_INET || 534 saidx->dst.sa.sa_family == AF_INET6, 535 "unexpected protocol family %u", saidx->dst.sa.sa_family); 536 537 esph = sav->tdb_authalgxform; 538 539 /* Check for crypto errors */ 540 if (crp->crp_etype) { 541 /* Reset the session ID */ 542 if (sav->tdb_cryptoid != 0) 543 sav->tdb_cryptoid = crp->crp_sid; 544 545 if (crp->crp_etype == EAGAIN) { 546 KEY_SA_UNREF(&sav); 547 IPSEC_RELEASE_GLOBAL_LOCKS(); 548 return crypto_dispatch(crp); 549 } 550 551 ESP_STATINC(ESP_STAT_NOXFORM); 552 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 553 error = crp->crp_etype; 554 goto bad; 555 } 556 557 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 558 559 /* If authentication was performed, check now. */ 560 if (esph != NULL) { 561 /* 562 * If we have a tag, it means an IPsec-aware NIC did 563 * the verification for us. Otherwise we need to 564 * check the authentication calculation. 565 */ 566 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 567 /* Copy the authenticator from the packet */ 568 m_copydata(m, m->m_pkthdr.len - esph->authsize, 569 esph->authsize, aalg); 570 571 ptr = (tc + 1); 572 573 /* Verify authenticator */ 574 if (!consttime_memequal(ptr, aalg, esph->authsize)) { 575 DPRINTF(("%s: authentication hash mismatch " 576 "for packet in SA %s/%08lx\n", __func__, 577 ipsec_address(&saidx->dst, buf, 578 sizeof(buf)), (u_long) ntohl(sav->spi))); 579 ESP_STATINC(ESP_STAT_BADAUTH); 580 error = EACCES; 581 goto bad; 582 } 583 584 /* Remove trailing authenticator */ 585 m_adj(m, -(esph->authsize)); 586 } 587 588 /* Release the crypto descriptors */ 589 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 590 tc = NULL; 591 crypto_freereq(crp); 592 crp = NULL; 593 594 /* 595 * Packet is now decrypted. 596 */ 597 m->m_flags |= M_DECRYPTED; 598 599 /* 600 * Update replay sequence number, if appropriate. 601 */ 602 if (sav->replay) { 603 uint32_t seq; 604 605 m_copydata(m, skip + offsetof(struct newesp, esp_seq), 606 sizeof(seq), &seq); 607 if (ipsec_updatereplay(ntohl(seq), sav)) { 608 char logbuf[IPSEC_LOGSASTRLEN]; 609 DPRINTF(("%s: packet replay check for %s\n", __func__, 610 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); 611 ESP_STATINC(ESP_STAT_REPLAY); 612 error = EACCES; 613 goto bad; 614 } 615 } 616 617 /* Determine the ESP header length */ 618 if (sav->flags & SADB_X_EXT_OLD) 619 hlen = sizeof(struct esp) + sav->ivlen; 620 else 621 hlen = sizeof(struct newesp) + sav->ivlen; 622 623 /* Remove the ESP header and IV from the mbuf. */ 624 error = m_striphdr(m, skip, hlen); 625 if (error) { 626 ESP_STATINC(ESP_STAT_HDROPS); 627 DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__, 628 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 629 (u_long) ntohl(sav->spi))); 630 goto bad; 631 } 632 633 /* Save the last three bytes of decrypted data */ 634 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree); 635 636 /* Verify pad length */ 637 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) { 638 ESP_STATINC(ESP_STAT_BADILEN); 639 DPRINTF(("%s: invalid padding length %d " 640 "for %u byte packet in SA %s/%08lx\n", __func__, 641 lastthree[1], m->m_pkthdr.len - skip, 642 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 643 (u_long) ntohl(sav->spi))); 644 error = EINVAL; 645 goto bad; 646 } 647 648 /* Verify correct decryption by checking the last padding bytes */ 649 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) { 650 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) { 651 ESP_STATINC(ESP_STAT_BADENC); 652 DPRINTF(("%s: decryption failed for packet in SA " 653 "%s/%08lx\n", __func__, 654 ipsec_address(&sav->sah->saidx.dst, buf, 655 sizeof(buf)), (u_long) ntohl(sav->spi))); 656 DPRINTF(("%s: %x %x\n", __func__, lastthree[0], 657 lastthree[1])); 658 error = EINVAL; 659 goto bad; 660 } 661 } 662 663 /* Trim the mbuf chain to remove trailing authenticator and padding */ 664 m_adj(m, -(lastthree[1] + 2)); 665 666 /* Restore the Next Protocol field */ 667 m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2); 668 669 IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff); 670 671 KEY_SA_UNREF(&sav); 672 IPSEC_RELEASE_GLOBAL_LOCKS(); 673 return error; 674 bad: 675 if (sav) 676 KEY_SA_UNREF(&sav); 677 IPSEC_RELEASE_GLOBAL_LOCKS(); 678 if (m != NULL) 679 m_freem(m); 680 if (tc != NULL) 681 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 682 if (crp != NULL) 683 crypto_freereq(crp); 684 return error; 685 } 686 687 /* 688 * ESP output routine, called by ipsec[46]_process_packet(). 689 */ 690 static int 691 esp_output(struct mbuf *m, const struct ipsecrequest *isr, struct secasvar *sav, 692 int skip, int protoff) 693 { 694 char buf[IPSEC_ADDRSTRLEN]; 695 const struct enc_xform *espx; 696 const struct auth_hash *esph; 697 int hlen, rlen, tlen, padlen, blks, alen, i, roff; 698 struct mbuf *mo = NULL; 699 struct tdb_crypto *tc; 700 struct secasindex *saidx; 701 unsigned char *tail; 702 uint8_t prot; 703 int error, maxpacketsize; 704 struct esptail *esptail; 705 struct cryptodesc *crde, *crda; 706 struct cryptop *crp; 707 708 esph = sav->tdb_authalgxform; 709 espx = sav->tdb_encalgxform; 710 KASSERT(espx != NULL); 711 712 /* Determine the ESP header length */ 713 if (sav->flags & SADB_X_EXT_OLD) 714 hlen = sizeof(struct esp) + sav->ivlen; 715 else 716 hlen = sizeof(struct newesp) + sav->ivlen; 717 /* Authenticator hash size */ 718 alen = esph ? esph->authsize : 0; 719 720 /* 721 * NB: The null encoding transform has a blocksize of 4 722 * so that headers are properly aligned. 723 */ 724 blks = espx->blocksize; /* IV blocksize */ 725 726 /* Raw payload length. */ 727 rlen = m->m_pkthdr.len - skip; 728 729 /* Encryption padding. */ 730 padlen = ((blks - ((rlen + sizeof(struct esptail)) % blks)) % blks); 731 732 /* Length of what we append (tail). */ 733 tlen = padlen + sizeof(struct esptail) + alen; 734 735 ESP_STATINC(ESP_STAT_OUTPUT); 736 737 saidx = &sav->sah->saidx; 738 /* Check for maximum packet size violations. */ 739 switch (saidx->dst.sa.sa_family) { 740 #ifdef INET 741 case AF_INET: 742 maxpacketsize = IP_MAXPACKET; 743 break; 744 #endif 745 #ifdef INET6 746 case AF_INET6: 747 maxpacketsize = IPV6_MAXPACKET; 748 break; 749 #endif 750 default: 751 DPRINTF(("%s: unknown/unsupported protocol family %d, " 752 "SA %s/%08lx\n", __func__, saidx->dst.sa.sa_family, 753 ipsec_address(&saidx->dst, buf, sizeof(buf)), 754 (u_long)ntohl(sav->spi))); 755 ESP_STATINC(ESP_STAT_NOPF); 756 error = EPFNOSUPPORT; 757 goto bad; 758 } 759 if (skip + hlen + rlen + tlen > maxpacketsize) { 760 DPRINTF(("%s: packet in SA %s/%08lx got too big (len %u, " 761 "max len %u)\n", __func__, 762 ipsec_address(&saidx->dst, buf, sizeof(buf)), 763 (u_long) ntohl(sav->spi), 764 skip + hlen + rlen + tlen, maxpacketsize)); 765 ESP_STATINC(ESP_STAT_TOOBIG); 766 error = EMSGSIZE; 767 goto bad; 768 } 769 770 /* Update the counters. */ 771 ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip); 772 773 m = m_clone(m); 774 if (m == NULL) { 775 DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__, 776 ipsec_address(&saidx->dst, buf, sizeof(buf)), 777 (u_long) ntohl(sav->spi))); 778 ESP_STATINC(ESP_STAT_HDROPS); 779 error = ENOBUFS; 780 goto bad; 781 } 782 783 /* Inject ESP header. */ 784 mo = m_makespace(m, skip, hlen, &roff); 785 if (mo == NULL) { 786 DPRINTF(("%s: failed to inject %u byte ESP hdr for SA " 787 "%s/%08lx\n", __func__, hlen, 788 ipsec_address(&saidx->dst, buf, sizeof(buf)), 789 (u_long) ntohl(sav->spi))); 790 ESP_STATINC(ESP_STAT_HDROPS); 791 error = ENOBUFS; 792 goto bad; 793 } 794 795 /* Initialize ESP header. */ 796 memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t)); 797 if (sav->replay) { 798 uint32_t replay; 799 800 #ifdef IPSEC_DEBUG 801 /* Emulate replay attack when ipsec_replay is TRUE. */ 802 if (!ipsec_replay) 803 #endif 804 sav->replay->count++; 805 806 replay = htonl(sav->replay->count); 807 memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay, 808 sizeof(uint32_t)); 809 } 810 811 /* 812 * Grow the mbuf, we will append data at the tail. 813 */ 814 tail = m_pad(m, tlen); 815 if (tail == NULL) { 816 DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__, 817 ipsec_address(&saidx->dst, buf, sizeof(buf)), 818 (u_long) ntohl(sav->spi))); 819 m = NULL; 820 error = ENOBUFS; 821 goto bad; 822 } 823 824 /* 825 * Add padding: random, zero, or self-describing. 826 */ 827 switch (sav->flags & SADB_X_EXT_PMASK) { 828 case SADB_X_EXT_PSEQ: 829 for (i = 0; i < padlen; i++) 830 tail[i] = i + 1; 831 break; 832 case SADB_X_EXT_PRAND: 833 (void)cprng_fast(tail, padlen); 834 break; 835 case SADB_X_EXT_PZERO: 836 default: 837 memset(tail, 0, padlen); 838 break; 839 } 840 841 /* Build the ESP Trailer. */ 842 esptail = (struct esptail *)&tail[padlen]; 843 esptail->esp_padlen = padlen; 844 m_copydata(m, protoff, sizeof(uint8_t), &esptail->esp_nxt); 845 846 /* Fix Next Protocol in IPv4/IPv6 header. */ 847 prot = IPPROTO_ESP; 848 m_copyback(m, protoff, sizeof(uint8_t), &prot); 849 850 /* Get crypto descriptors. */ 851 crp = crypto_getreq(esph ? 2 : 1); 852 if (crp == NULL) { 853 DPRINTF(("%s: failed to acquire crypto descriptors\n", 854 __func__)); 855 ESP_STATINC(ESP_STAT_CRYPTO); 856 error = ENOBUFS; 857 goto bad; 858 } 859 860 /* Get the descriptors. */ 861 crde = crp->crp_desc; 862 crda = crde->crd_next; 863 864 /* Encryption descriptor. */ 865 crde->crd_skip = skip + hlen; 866 if (espx->type == CRYPTO_AES_GMAC) 867 crde->crd_len = 0; 868 else 869 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 870 crde->crd_flags = CRD_F_ENCRYPT; 871 crde->crd_inject = skip + hlen - sav->ivlen; 872 crde->crd_alg = espx->type; 873 crde->crd_key = _KEYBUF(sav->key_enc); 874 crde->crd_klen = _KEYBITS(sav->key_enc); 875 /* XXX Rounds ? */ 876 877 /* IPsec-specific opaque crypto info. */ 878 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 879 if (tc == NULL) { 880 crypto_freereq(crp); 881 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); 882 ESP_STATINC(ESP_STAT_CRYPTO); 883 error = ENOBUFS; 884 goto bad; 885 } 886 887 { 888 int s = pserialize_read_enter(); 889 890 /* 891 * Take another reference to the SP and the SA for opencrypto callback. 892 */ 893 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD || 894 sav->state == SADB_SASTATE_DEAD)) { 895 pserialize_read_exit(s); 896 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 897 crypto_freereq(crp); 898 ESP_STATINC(ESP_STAT_NOTDB); 899 error = ENOENT; 900 goto bad; 901 } 902 KEY_SP_REF(isr->sp); 903 KEY_SA_REF(sav); 904 pserialize_read_exit(s); 905 } 906 907 /* Callback parameters */ 908 tc->tc_isr = isr; 909 tc->tc_spi = sav->spi; 910 tc->tc_dst = saidx->dst; 911 tc->tc_proto = saidx->proto; 912 tc->tc_sav = sav; 913 914 /* Crypto operation descriptor. */ 915 crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ 916 crp->crp_flags = CRYPTO_F_IMBUF; 917 crp->crp_buf = m; 918 crp->crp_callback = esp_output_cb; 919 crp->crp_opaque = tc; 920 crp->crp_sid = sav->tdb_cryptoid; 921 922 if (esph) { 923 /* Authentication descriptor. */ 924 crda->crd_skip = skip; 925 if (espx->type == CRYPTO_AES_GCM_16) 926 crda->crd_len = hlen - sav->ivlen; 927 else 928 crda->crd_len = m->m_pkthdr.len - (skip + alen); 929 crda->crd_inject = m->m_pkthdr.len - alen; 930 931 /* Authentication operation. */ 932 crda->crd_alg = esph->type; 933 if (espx->type == CRYPTO_AES_GCM_16 || 934 espx->type == CRYPTO_AES_GMAC) { 935 crda->crd_key = _KEYBUF(sav->key_enc); 936 crda->crd_klen = _KEYBITS(sav->key_enc); 937 } else { 938 crda->crd_key = _KEYBUF(sav->key_auth); 939 crda->crd_klen = _KEYBITS(sav->key_auth); 940 } 941 } 942 943 return crypto_dispatch(crp); 944 945 bad: 946 if (m) 947 m_freem(m); 948 return error; 949 } 950 951 /* 952 * ESP output callback from the crypto driver. 953 */ 954 static int 955 esp_output_cb(struct cryptop *crp) 956 { 957 struct tdb_crypto *tc; 958 const struct ipsecrequest *isr; 959 struct secasvar *sav; 960 struct mbuf *m; 961 int err, error; 962 IPSEC_DECLARE_LOCK_VARIABLE; 963 964 KASSERT(crp->crp_opaque != NULL); 965 tc = crp->crp_opaque; 966 m = crp->crp_buf; 967 968 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 969 970 isr = tc->tc_isr; 971 sav = tc->tc_sav; 972 973 /* Check for crypto errors. */ 974 if (crp->crp_etype) { 975 /* Reset session ID. */ 976 if (sav->tdb_cryptoid != 0) 977 sav->tdb_cryptoid = crp->crp_sid; 978 979 if (crp->crp_etype == EAGAIN) { 980 IPSEC_RELEASE_GLOBAL_LOCKS(); 981 return crypto_dispatch(crp); 982 } 983 984 ESP_STATINC(ESP_STAT_NOXFORM); 985 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 986 error = crp->crp_etype; 987 goto bad; 988 } 989 990 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 991 if (sav->tdb_authalgxform != NULL) 992 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 993 994 /* Release crypto descriptors. */ 995 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 996 crypto_freereq(crp); 997 998 #ifdef IPSEC_DEBUG 999 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */ 1000 if (ipsec_integrity) { 1001 static unsigned char ipseczeroes[AH_ALEN_MAX]; 1002 const struct auth_hash *esph; 1003 1004 /* 1005 * Corrupt HMAC if we want to test integrity verification of 1006 * the other side. 1007 */ 1008 esph = sav->tdb_authalgxform; 1009 if (esph != NULL) { 1010 m_copyback(m, m->m_pkthdr.len - esph->authsize, 1011 esph->authsize, ipseczeroes); 1012 } 1013 } 1014 #endif 1015 1016 /* NB: m is reclaimed by ipsec_process_done. */ 1017 err = ipsec_process_done(m, isr, sav); 1018 KEY_SA_UNREF(&sav); 1019 KEY_SP_UNREF(&isr->sp); 1020 IPSEC_RELEASE_GLOBAL_LOCKS(); 1021 return err; 1022 1023 bad: 1024 if (sav) 1025 KEY_SA_UNREF(&sav); 1026 KEY_SP_UNREF(&isr->sp); 1027 IPSEC_RELEASE_GLOBAL_LOCKS(); 1028 if (m) 1029 m_freem(m); 1030 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 1031 crypto_freereq(crp); 1032 return error; 1033 } 1034 1035 static struct xformsw esp_xformsw = { 1036 .xf_type = XF_ESP, 1037 .xf_flags = XFT_CONF|XFT_AUTH, 1038 .xf_name = "IPsec ESP", 1039 .xf_init = esp_init, 1040 .xf_zeroize = esp_zeroize, 1041 .xf_input = esp_input, 1042 .xf_output = esp_output, 1043 .xf_next = NULL, 1044 }; 1045 1046 void 1047 esp_attach(void) 1048 { 1049 1050 espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS); 1051 1052 extern int ah_max_authsize; 1053 KASSERT(ah_max_authsize != 0); 1054 esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize; 1055 esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size, 1056 coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET, 1057 NULL, NULL, NULL); 1058 1059 #define MAXIV(xform) \ 1060 if (xform.ivsize > esp_max_ivlen) \ 1061 esp_max_ivlen = xform.ivsize \ 1062 1063 esp_max_ivlen = 0; 1064 MAXIV(enc_xform_des); /* SADB_EALG_DESCBC */ 1065 MAXIV(enc_xform_3des); /* SADB_EALG_3DESCBC */ 1066 MAXIV(enc_xform_rijndael128); /* SADB_X_EALG_AES */ 1067 MAXIV(enc_xform_blf); /* SADB_X_EALG_BLOWFISHCBC */ 1068 MAXIV(enc_xform_cast5); /* SADB_X_EALG_CAST128CBC */ 1069 MAXIV(enc_xform_skipjack); /* SADB_X_EALG_SKIPJACK */ 1070 MAXIV(enc_xform_camellia); /* SADB_X_EALG_CAMELLIACBC */ 1071 MAXIV(enc_xform_aes_ctr); /* SADB_X_EALG_AESCTR */ 1072 MAXIV(enc_xform_null); /* SADB_EALG_NULL */ 1073 1074 xform_register(&esp_xformsw); 1075 #undef MAXIV 1076 } 1077