1 /* $NetBSD: xform_esp.c,v 1.72 2017/10/03 08:56:52 ozaki-r Exp $ */ 2 /* $FreeBSD: src/sys/netipsec/xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */ 3 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ 4 5 /* 6 * The authors of this code are John Ioannidis (ji@tla.org), 7 * Angelos D. Keromytis (kermit@csd.uch.gr) and 8 * Niels Provos (provos@physnet.uni-hamburg.de). 9 * 10 * The original version of this code was written by John Ioannidis 11 * for BSD/OS in Athens, Greece, in November 1995. 12 * 13 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, 14 * by Angelos D. Keromytis. 15 * 16 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis 17 * and Niels Provos. 18 * 19 * Additional features in 1999 by Angelos D. Keromytis. 20 * 21 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, 22 * Angelos D. Keromytis and Niels Provos. 23 * Copyright (c) 2001 Angelos D. Keromytis. 24 * 25 * Permission to use, copy, and modify this software with or without fee 26 * is hereby granted, provided that this entire notice is included in 27 * all copies of any software which is or includes a copy or 28 * modification of this software. 29 * You may use this code under the GNU public license if you so wish. Please 30 * contribute changes back to the authors under this freer than GPL license 31 * so that we may further the use of strong encryption without limitations to 32 * all. 33 * 34 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 35 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 36 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 37 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 38 * PURPOSE. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.72 2017/10/03 08:56:52 ozaki-r Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_inet.h" 46 #include "opt_ipsec.h" 47 #endif 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/mbuf.h> 52 #include <sys/socket.h> 53 #include <sys/syslog.h> 54 #include <sys/kernel.h> 55 #include <sys/sysctl.h> 56 #include <sys/cprng.h> 57 #include <sys/pool.h> 58 #include <sys/pserialize.h> 59 60 #include <net/if.h> 61 62 #include <netinet/in.h> 63 #include <netinet/in_systm.h> 64 #include <netinet/ip.h> 65 #include <netinet/ip_ecn.h> 66 #include <netinet/ip6.h> 67 68 #include <net/route.h> 69 #include <netipsec/ipsec.h> 70 #include <netipsec/ipsec_private.h> 71 #include <netipsec/ah.h> 72 #include <netipsec/ah_var.h> 73 #include <netipsec/esp.h> 74 #include <netipsec/esp_var.h> 75 #include <netipsec/xform.h> 76 77 #ifdef INET6 78 #include <netinet6/ip6_var.h> 79 #include <netipsec/ipsec6.h> 80 #endif 81 82 #include <netipsec/key.h> 83 #include <netipsec/key_debug.h> 84 85 #include <opencrypto/cryptodev.h> 86 #include <opencrypto/xform.h> 87 88 percpu_t *espstat_percpu; 89 90 int esp_enable = 1; 91 92 #ifdef __FreeBSD__ 93 SYSCTL_DECL(_net_inet_esp); 94 SYSCTL_INT(_net_inet_esp, OID_AUTO, 95 esp_enable, CTLFLAG_RW, &esp_enable, 0, ""); 96 SYSCTL_STRUCT(_net_inet_esp, IPSECCTL_STATS, 97 stats, CTLFLAG_RD, &espstat, espstat, ""); 98 #endif /* __FreeBSD__ */ 99 100 static int esp_max_ivlen; /* max iv length over all algorithms */ 101 102 static int esp_input_cb(struct cryptop *op); 103 static int esp_output_cb(struct cryptop *crp); 104 105 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT }; 106 107 static pool_cache_t esp_tdb_crypto_pool_cache; 108 static size_t esp_pool_item_size; 109 110 /* 111 * NB: this is public for use by the PF_KEY support. 112 * NB: if you add support here; be sure to add code to esp_attach below! 113 */ 114 const struct enc_xform * 115 esp_algorithm_lookup(int alg) 116 { 117 118 switch (alg) { 119 case SADB_EALG_DESCBC: 120 return &enc_xform_des; 121 case SADB_EALG_3DESCBC: 122 return &enc_xform_3des; 123 case SADB_X_EALG_AES: 124 return &enc_xform_rijndael128; 125 case SADB_X_EALG_BLOWFISHCBC: 126 return &enc_xform_blf; 127 case SADB_X_EALG_CAST128CBC: 128 return &enc_xform_cast5; 129 case SADB_X_EALG_SKIPJACK: 130 return &enc_xform_skipjack; 131 case SADB_X_EALG_CAMELLIACBC: 132 return &enc_xform_camellia; 133 case SADB_X_EALG_AESCTR: 134 return &enc_xform_aes_ctr; 135 case SADB_X_EALG_AESGCM16: 136 return &enc_xform_aes_gcm; 137 case SADB_X_EALG_AESGMAC: 138 return &enc_xform_aes_gmac; 139 case SADB_EALG_NULL: 140 return &enc_xform_null; 141 } 142 return NULL; 143 } 144 145 size_t 146 esp_hdrsiz(const struct secasvar *sav) 147 { 148 size_t size; 149 150 if (sav != NULL) { 151 /*XXX not right for null algorithm--does it matter??*/ 152 KASSERT(sav->tdb_encalgxform != NULL); 153 if (sav->flags & SADB_X_EXT_OLD) 154 size = sizeof(struct esp); 155 else 156 size = sizeof(struct newesp); 157 size += sav->tdb_encalgxform->ivsize + 9; 158 /*XXX need alg check???*/ 159 if (sav->tdb_authalgxform != NULL && sav->replay) 160 size += ah_hdrsiz(sav); 161 } else { 162 /* 163 * base header size 164 * + max iv length for CBC mode 165 * + max pad length 166 * + sizeof(pad length field) 167 * + sizeof(next header field) 168 * + max icv supported. 169 */ 170 size = sizeof(struct newesp) + esp_max_ivlen + 9 + 171 ah_hdrsiz(NULL); 172 } 173 return size; 174 } 175 176 /* 177 * esp_init() is called when an SPI is being set up. 178 */ 179 static int 180 esp_init(struct secasvar *sav, const struct xformsw *xsp) 181 { 182 const struct enc_xform *txform; 183 struct cryptoini cria, crie, *cr; 184 int keylen; 185 int error; 186 187 txform = esp_algorithm_lookup(sav->alg_enc); 188 if (txform == NULL) { 189 DPRINTF(("%s: unsupported encryption algorithm %d\n", __func__, 190 sav->alg_enc)); 191 return EINVAL; 192 } 193 if (sav->key_enc == NULL) { 194 DPRINTF(("%s: no encoding key for %s algorithm\n", __func__, 195 txform->name)); 196 return EINVAL; 197 } 198 if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) { 199 DPRINTF(("%s: 4-byte IV not supported with protocol\n", 200 __func__)); 201 return EINVAL; 202 } 203 keylen = _KEYLEN(sav->key_enc); 204 if (txform->minkey > keylen || keylen > txform->maxkey) { 205 DPRINTF(("%s: invalid key length %u, must be in " 206 "the range [%u..%u] for algorithm %s\n", __func__, 207 keylen, txform->minkey, txform->maxkey, txform->name)); 208 return EINVAL; 209 } 210 211 sav->ivlen = txform->ivsize; 212 213 /* 214 * Setup AH-related state. 215 */ 216 if (sav->alg_auth != 0) { 217 error = ah_init0(sav, xsp, &cria); 218 if (error) 219 return error; 220 } 221 222 /* NB: override anything set in ah_init0 */ 223 sav->tdb_xform = xsp; 224 sav->tdb_encalgxform = txform; 225 226 switch (sav->alg_enc) { 227 case SADB_X_EALG_AESGCM16: 228 case SADB_X_EALG_AESGMAC: 229 switch (keylen) { 230 case 20: 231 sav->alg_auth = SADB_X_AALG_AES128GMAC; 232 sav->tdb_authalgxform = &auth_hash_gmac_aes_128; 233 break; 234 case 28: 235 sav->alg_auth = SADB_X_AALG_AES192GMAC; 236 sav->tdb_authalgxform = &auth_hash_gmac_aes_192; 237 break; 238 case 36: 239 sav->alg_auth = SADB_X_AALG_AES256GMAC; 240 sav->tdb_authalgxform = &auth_hash_gmac_aes_256; 241 break; 242 default: 243 DPRINTF(("%s: invalid key length %u, must be either of " 244 "20, 28 or 36\n", __func__, keylen)); 245 return EINVAL; 246 } 247 248 memset(&cria, 0, sizeof(cria)); 249 cria.cri_alg = sav->tdb_authalgxform->type; 250 cria.cri_klen = _KEYBITS(sav->key_enc); 251 cria.cri_key = _KEYBUF(sav->key_enc); 252 break; 253 default: 254 break; 255 } 256 257 /* Initialize crypto session. */ 258 memset(&crie, 0, sizeof(crie)); 259 crie.cri_alg = sav->tdb_encalgxform->type; 260 crie.cri_klen = _KEYBITS(sav->key_enc); 261 crie.cri_key = _KEYBUF(sav->key_enc); 262 /* XXX Rounds ? */ 263 264 if (sav->tdb_authalgxform && sav->tdb_encalgxform) { 265 /* init both auth & enc */ 266 crie.cri_next = &cria; 267 cr = &crie; 268 } else if (sav->tdb_encalgxform) { 269 cr = &crie; 270 } else if (sav->tdb_authalgxform) { 271 cr = &cria; 272 } else { 273 /* XXX cannot happen? */ 274 DPRINTF(("%s: no encoding OR authentication xform!\n", 275 __func__)); 276 return EINVAL; 277 } 278 279 return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support); 280 } 281 282 /* 283 * Paranoia. 284 */ 285 static int 286 esp_zeroize(struct secasvar *sav) 287 { 288 /* NB: ah_zerorize free's the crypto session state */ 289 int error = ah_zeroize(sav); 290 291 if (sav->key_enc) { 292 explicit_memset(_KEYBUF(sav->key_enc), 0, 293 _KEYLEN(sav->key_enc)); 294 } 295 sav->tdb_encalgxform = NULL; 296 sav->tdb_xform = NULL; 297 return error; 298 } 299 300 /* 301 * ESP input processing, called (eventually) through the protocol switch. 302 */ 303 static int 304 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) 305 { 306 const struct auth_hash *esph; 307 const struct enc_xform *espx; 308 struct tdb_crypto *tc; 309 int plen, alen, hlen, error; 310 struct newesp *esp; 311 312 struct cryptodesc *crde; 313 struct cryptop *crp; 314 315 IPSEC_SPLASSERT_SOFTNET(__func__); 316 317 KASSERT(sav != NULL); 318 KASSERT(sav->tdb_encalgxform != NULL); 319 KASSERTMSG((skip&3) == 0 && (m->m_pkthdr.len&3) == 0, 320 "misaligned packet, skip %u pkt len %u", 321 skip, m->m_pkthdr.len); 322 323 /* XXX don't pullup, just copy header */ 324 IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof(struct newesp)); 325 326 esph = sav->tdb_authalgxform; 327 espx = sav->tdb_encalgxform; 328 329 /* Determine the ESP header length */ 330 if (sav->flags & SADB_X_EXT_OLD) 331 hlen = sizeof(struct esp) + sav->ivlen; 332 else 333 hlen = sizeof(struct newesp) + sav->ivlen; 334 /* Authenticator hash size */ 335 alen = esph ? esph->authsize : 0; 336 337 /* 338 * Verify payload length is multiple of encryption algorithm 339 * block size. 340 * 341 * NB: This works for the null algorithm because the blocksize 342 * is 4 and all packets must be 4-byte aligned regardless 343 * of the algorithm. 344 */ 345 plen = m->m_pkthdr.len - (skip + hlen + alen); 346 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) { 347 char buf[IPSEC_ADDRSTRLEN]; 348 DPRINTF(("%s: payload of %d octets not a multiple of %d octets," 349 " SA %s/%08lx\n", __func__, plen, espx->blocksize, 350 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 351 (u_long) ntohl(sav->spi))); 352 ESP_STATINC(ESP_STAT_BADILEN); 353 m_freem(m); 354 return EINVAL; 355 } 356 357 /* 358 * Check sequence number. 359 */ 360 if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) { 361 char logbuf[IPSEC_LOGSASTRLEN]; 362 DPRINTF(("%s: packet replay check for %s\n", __func__, 363 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); /*XXX*/ 364 ESP_STATINC(ESP_STAT_REPLAY); 365 m_freem(m); 366 return ENOBUFS; /*XXX*/ 367 } 368 369 /* Update the counters */ 370 ESP_STATADD(ESP_STAT_IBYTES, m->m_pkthdr.len - skip - hlen - alen); 371 372 /* Get crypto descriptors */ 373 crp = crypto_getreq(esph && espx ? 2 : 1); 374 if (crp == NULL) { 375 DPRINTF(("%s: failed to acquire crypto descriptors\n", 376 __func__)); 377 error = ENOBUFS; 378 goto out; 379 } 380 381 /* Get IPsec-specific opaque pointer */ 382 size_t extra __diagused = esph == NULL ? 0 : alen; 383 KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size, 384 "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n", 385 sizeof(*tc) + extra, esp_pool_item_size); 386 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 387 if (tc == NULL) { 388 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); 389 error = ENOBUFS; 390 goto out1; 391 } 392 393 error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT); 394 if (error) { 395 DPRINTF(("%s: m_makewritable failed\n", __func__)); 396 goto out2; 397 } 398 399 if (esph) { 400 struct cryptodesc *crda; 401 402 KASSERT(crp->crp_desc != NULL); 403 crda = crp->crp_desc; 404 405 /* Authentication descriptor */ 406 crda->crd_skip = skip; 407 if (espx && espx->type == CRYPTO_AES_GCM_16) 408 crda->crd_len = hlen - sav->ivlen; 409 else 410 crda->crd_len = m->m_pkthdr.len - (skip + alen); 411 crda->crd_inject = m->m_pkthdr.len - alen; 412 413 crda->crd_alg = esph->type; 414 if (espx && (espx->type == CRYPTO_AES_GCM_16 || 415 espx->type == CRYPTO_AES_GMAC)) { 416 crda->crd_key = _KEYBUF(sav->key_enc); 417 crda->crd_klen = _KEYBITS(sav->key_enc); 418 } else { 419 crda->crd_key = _KEYBUF(sav->key_auth); 420 crda->crd_klen = _KEYBITS(sav->key_auth); 421 } 422 423 /* Copy the authenticator */ 424 m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1)); 425 426 /* Chain authentication request */ 427 crde = crda->crd_next; 428 } else { 429 crde = crp->crp_desc; 430 } 431 432 { 433 int s = pserialize_read_enter(); 434 435 /* 436 * Take another reference to the SA for opencrypto callback. 437 */ 438 if (__predict_false(sav->state == SADB_SASTATE_DEAD)) { 439 pserialize_read_exit(s); 440 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 441 crypto_freereq(crp); 442 ESP_STATINC(ESP_STAT_NOTDB); 443 return ENOENT; 444 } 445 KEY_SA_REF(sav); 446 pserialize_read_exit(s); 447 } 448 449 /* Crypto operation descriptor */ 450 crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ 451 crp->crp_flags = CRYPTO_F_IMBUF; 452 crp->crp_buf = m; 453 crp->crp_callback = esp_input_cb; 454 crp->crp_sid = sav->tdb_cryptoid; 455 crp->crp_opaque = tc; 456 457 /* These are passed as-is to the callback */ 458 tc->tc_spi = sav->spi; 459 tc->tc_dst = sav->sah->saidx.dst; 460 tc->tc_proto = sav->sah->saidx.proto; 461 tc->tc_protoff = protoff; 462 tc->tc_skip = skip; 463 tc->tc_sav = sav; 464 465 /* Decryption descriptor */ 466 if (espx) { 467 KASSERTMSG(crde != NULL, "null esp crypto descriptor"); 468 crde->crd_skip = skip + hlen; 469 if (espx->type == CRYPTO_AES_GMAC) 470 crde->crd_len = 0; 471 else 472 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 473 crde->crd_inject = skip + hlen - sav->ivlen; 474 475 crde->crd_alg = espx->type; 476 crde->crd_key = _KEYBUF(sav->key_enc); 477 crde->crd_klen = _KEYBITS(sav->key_enc); 478 /* XXX Rounds ? */ 479 } 480 481 return crypto_dispatch(crp); 482 483 out2: 484 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 485 out1: 486 crypto_freereq(crp); 487 out: 488 ESP_STATINC(ESP_STAT_CRYPTO); 489 m_freem(m); 490 return error; 491 } 492 493 #ifdef INET6 494 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do { \ 495 if (saidx->dst.sa.sa_family == AF_INET6) { \ 496 error = ipsec6_common_input_cb(m, sav, skip, protoff); \ 497 } else { \ 498 error = ipsec4_common_input_cb(m, sav, skip, protoff); \ 499 } \ 500 } while (0) 501 #else 502 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) \ 503 (error = ipsec4_common_input_cb(m, sav, skip, protoff)) 504 #endif 505 506 /* 507 * ESP input callback from the crypto driver. 508 */ 509 static int 510 esp_input_cb(struct cryptop *crp) 511 { 512 char buf[IPSEC_ADDRSTRLEN]; 513 uint8_t lastthree[3], aalg[AH_ALEN_MAX]; 514 int hlen, skip, protoff, error; 515 struct mbuf *m; 516 const struct auth_hash *esph; 517 struct tdb_crypto *tc; 518 struct secasvar *sav; 519 struct secasindex *saidx; 520 void *ptr; 521 uint16_t dport; 522 uint16_t sport; 523 IPSEC_DECLARE_LOCK_VARIABLE; 524 525 KASSERT(crp->crp_desc != NULL); 526 KASSERT(crp->crp_opaque != NULL); 527 528 tc = crp->crp_opaque; 529 skip = tc->tc_skip; 530 protoff = tc->tc_protoff; 531 m = crp->crp_buf; 532 533 /* find the source port for NAT-T */ 534 nat_t_ports_get(m, &dport, &sport); 535 536 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 537 538 sav = tc->tc_sav; 539 if (__predict_false(!SADB_SASTATE_USABLE_P(sav))) { 540 KEY_SA_UNREF(&sav); 541 sav = KEY_LOOKUP_SA(&tc->tc_dst, tc->tc_proto, tc->tc_spi, 542 sport, dport); 543 if (sav == NULL) { 544 ESP_STATINC(ESP_STAT_NOTDB); 545 DPRINTF(("%s: SA expired while in crypto " 546 "(SA %s/%08lx proto %u)\n", __func__, 547 ipsec_address(&tc->tc_dst, buf, sizeof(buf)), 548 (u_long) ntohl(tc->tc_spi), tc->tc_proto)); 549 error = ENOBUFS; /*XXX*/ 550 goto bad; 551 } 552 } 553 554 saidx = &sav->sah->saidx; 555 KASSERTMSG(saidx->dst.sa.sa_family == AF_INET || 556 saidx->dst.sa.sa_family == AF_INET6, 557 "unexpected protocol family %u", saidx->dst.sa.sa_family); 558 559 esph = sav->tdb_authalgxform; 560 561 /* Check for crypto errors */ 562 if (crp->crp_etype) { 563 /* Reset the session ID */ 564 if (sav->tdb_cryptoid != 0) 565 sav->tdb_cryptoid = crp->crp_sid; 566 567 if (crp->crp_etype == EAGAIN) { 568 KEY_SA_UNREF(&sav); 569 IPSEC_RELEASE_GLOBAL_LOCKS(); 570 return crypto_dispatch(crp); 571 } 572 573 ESP_STATINC(ESP_STAT_NOXFORM); 574 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 575 error = crp->crp_etype; 576 goto bad; 577 } 578 579 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 580 581 /* If authentication was performed, check now. */ 582 if (esph != NULL) { 583 /* 584 * If we have a tag, it means an IPsec-aware NIC did 585 * the verification for us. Otherwise we need to 586 * check the authentication calculation. 587 */ 588 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 589 /* Copy the authenticator from the packet */ 590 m_copydata(m, m->m_pkthdr.len - esph->authsize, 591 esph->authsize, aalg); 592 593 ptr = (tc + 1); 594 595 /* Verify authenticator */ 596 if (!consttime_memequal(ptr, aalg, esph->authsize)) { 597 DPRINTF(("%s: authentication hash mismatch " 598 "for packet in SA %s/%08lx\n", __func__, 599 ipsec_address(&saidx->dst, buf, 600 sizeof(buf)), (u_long) ntohl(sav->spi))); 601 ESP_STATINC(ESP_STAT_BADAUTH); 602 error = EACCES; 603 goto bad; 604 } 605 606 /* Remove trailing authenticator */ 607 m_adj(m, -(esph->authsize)); 608 } 609 610 /* Release the crypto descriptors */ 611 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 612 tc = NULL; 613 crypto_freereq(crp), crp = NULL; 614 615 /* 616 * Packet is now decrypted. 617 */ 618 m->m_flags |= M_DECRYPTED; 619 620 /* 621 * Update replay sequence number, if appropriate. 622 */ 623 if (sav->replay) { 624 uint32_t seq; 625 626 m_copydata(m, skip + offsetof(struct newesp, esp_seq), 627 sizeof(seq), &seq); 628 if (ipsec_updatereplay(ntohl(seq), sav)) { 629 char logbuf[IPSEC_LOGSASTRLEN]; 630 DPRINTF(("%s: packet replay check for %s\n", __func__, 631 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); 632 ESP_STATINC(ESP_STAT_REPLAY); 633 error = ENOBUFS; 634 goto bad; 635 } 636 } 637 638 /* Determine the ESP header length */ 639 if (sav->flags & SADB_X_EXT_OLD) 640 hlen = sizeof(struct esp) + sav->ivlen; 641 else 642 hlen = sizeof(struct newesp) + sav->ivlen; 643 644 /* Remove the ESP header and IV from the mbuf. */ 645 error = m_striphdr(m, skip, hlen); 646 if (error) { 647 ESP_STATINC(ESP_STAT_HDROPS); 648 DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__, 649 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 650 (u_long) ntohl(sav->spi))); 651 goto bad; 652 } 653 654 /* Save the last three bytes of decrypted data */ 655 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree); 656 657 /* Verify pad length */ 658 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) { 659 ESP_STATINC(ESP_STAT_BADILEN); 660 DPRINTF(("%s: invalid padding length %d " 661 "for %u byte packet in SA %s/%08lx\n", __func__, 662 lastthree[1], m->m_pkthdr.len - skip, 663 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 664 (u_long) ntohl(sav->spi))); 665 error = EINVAL; 666 goto bad; 667 } 668 669 /* Verify correct decryption by checking the last padding bytes */ 670 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) { 671 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) { 672 ESP_STATINC(ESP_STAT_BADENC); 673 DPRINTF(("%s: decryption failed for packet in SA " 674 "%s/%08lx\n", __func__, 675 ipsec_address(&sav->sah->saidx.dst, buf, 676 sizeof(buf)), (u_long) ntohl(sav->spi))); 677 DPRINTF(("%s: %x %x\n", __func__, lastthree[0], 678 lastthree[1])); 679 error = EINVAL; 680 goto bad; 681 } 682 } 683 684 /* Trim the mbuf chain to remove trailing authenticator and padding */ 685 m_adj(m, -(lastthree[1] + 2)); 686 687 /* Restore the Next Protocol field */ 688 m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2); 689 690 IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff); 691 692 KEY_SA_UNREF(&sav); 693 IPSEC_RELEASE_GLOBAL_LOCKS(); 694 return error; 695 bad: 696 if (sav) 697 KEY_SA_UNREF(&sav); 698 IPSEC_RELEASE_GLOBAL_LOCKS(); 699 if (m != NULL) 700 m_freem(m); 701 if (tc != NULL) 702 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 703 if (crp != NULL) 704 crypto_freereq(crp); 705 return error; 706 } 707 708 /* 709 * ESP output routine, called by ipsec[46]_process_packet(). 710 */ 711 static int 712 esp_output( 713 struct mbuf *m, 714 const struct ipsecrequest *isr, 715 struct secasvar *sav, 716 struct mbuf **mp, 717 int skip, 718 int protoff 719 ) 720 { 721 char buf[IPSEC_ADDRSTRLEN]; 722 const struct enc_xform *espx; 723 const struct auth_hash *esph; 724 int hlen, rlen, padding, blks, alen, i, roff; 725 struct mbuf *mo = NULL; 726 struct tdb_crypto *tc; 727 struct secasindex *saidx; 728 unsigned char *pad; 729 uint8_t prot; 730 int error, maxpacketsize; 731 732 struct cryptodesc *crde = NULL, *crda = NULL; 733 struct cryptop *crp; 734 735 IPSEC_SPLASSERT_SOFTNET(__func__); 736 737 esph = sav->tdb_authalgxform; 738 KASSERT(sav->tdb_encalgxform != NULL); 739 espx = sav->tdb_encalgxform; 740 741 if (sav->flags & SADB_X_EXT_OLD) 742 hlen = sizeof(struct esp) + sav->ivlen; 743 else 744 hlen = sizeof(struct newesp) + sav->ivlen; 745 746 rlen = m->m_pkthdr.len - skip; /* Raw payload length. */ 747 /* 748 * NB: The null encoding transform has a blocksize of 4 749 * so that headers are properly aligned. 750 */ 751 blks = espx->blocksize; /* IV blocksize */ 752 753 /* XXX clamp padding length a la KAME??? */ 754 padding = ((blks - ((rlen + 2) % blks)) % blks) + 2; 755 756 if (esph) 757 alen = esph->authsize; 758 else 759 alen = 0; 760 761 ESP_STATINC(ESP_STAT_OUTPUT); 762 763 saidx = &sav->sah->saidx; 764 /* Check for maximum packet size violations. */ 765 switch (saidx->dst.sa.sa_family) { 766 #ifdef INET 767 case AF_INET: 768 maxpacketsize = IP_MAXPACKET; 769 break; 770 #endif /* INET */ 771 #ifdef INET6 772 case AF_INET6: 773 maxpacketsize = IPV6_MAXPACKET; 774 break; 775 #endif /* INET6 */ 776 default: 777 DPRINTF(("%s: unknown/unsupported protocol family %d, " 778 "SA %s/%08lx\n", __func__, saidx->dst.sa.sa_family, 779 ipsec_address(&saidx->dst, buf, sizeof(buf)), 780 (u_long)ntohl(sav->spi))); 781 ESP_STATINC(ESP_STAT_NOPF); 782 error = EPFNOSUPPORT; 783 goto bad; 784 } 785 if (skip + hlen + rlen + padding + alen > maxpacketsize) { 786 DPRINTF(("%s: packet in SA %s/%08lx got too big (len %u, " 787 "max len %u)\n", __func__, 788 ipsec_address(&saidx->dst, buf, sizeof(buf)), 789 (u_long) ntohl(sav->spi), 790 skip + hlen + rlen + padding + alen, maxpacketsize)); 791 ESP_STATINC(ESP_STAT_TOOBIG); 792 error = EMSGSIZE; 793 goto bad; 794 } 795 796 /* Update the counters. */ 797 ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip); 798 799 m = m_clone(m); 800 if (m == NULL) { 801 DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__, 802 ipsec_address(&saidx->dst, buf, sizeof(buf)), 803 (u_long) ntohl(sav->spi))); 804 ESP_STATINC(ESP_STAT_HDROPS); 805 error = ENOBUFS; 806 goto bad; 807 } 808 809 /* Inject ESP header. */ 810 mo = m_makespace(m, skip, hlen, &roff); 811 if (mo == NULL) { 812 DPRINTF(("%s: failed to inject %u byte ESP hdr for SA " 813 "%s/%08lx\n", __func__, hlen, 814 ipsec_address(&saidx->dst, buf, sizeof(buf)), 815 (u_long) ntohl(sav->spi))); 816 ESP_STATINC(ESP_STAT_HDROPS); /* XXX diffs from openbsd */ 817 error = ENOBUFS; 818 goto bad; 819 } 820 821 /* Initialize ESP header. */ 822 memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t)); 823 if (sav->replay) { 824 uint32_t replay; 825 826 #ifdef IPSEC_DEBUG 827 /* Emulate replay attack when ipsec_replay is TRUE. */ 828 if (!ipsec_replay) 829 #endif 830 sav->replay->count++; 831 832 replay = htonl(sav->replay->count); 833 memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay, 834 sizeof(uint32_t)); 835 } 836 837 /* 838 * Add padding -- better to do it ourselves than use the crypto engine, 839 * although if/when we support compression, we'd have to do that. 840 */ 841 pad = m_pad(m, padding + alen); 842 if (pad == NULL) { 843 DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__, 844 ipsec_address(&saidx->dst, buf, sizeof(buf)), 845 (u_long) ntohl(sav->spi))); 846 m = NULL; /* NB: free'd by m_pad */ 847 error = ENOBUFS; 848 goto bad; 849 } 850 851 /* 852 * Add padding: random, zero, or self-describing. 853 * XXX catch unexpected setting 854 */ 855 switch (sav->flags & SADB_X_EXT_PMASK) { 856 case SADB_X_EXT_PRAND: 857 (void) cprng_fast(pad, padding - 2); 858 break; 859 case SADB_X_EXT_PZERO: 860 memset(pad, 0, padding - 2); 861 break; 862 case SADB_X_EXT_PSEQ: 863 for (i = 0; i < padding - 2; i++) 864 pad[i] = i+1; 865 break; 866 } 867 868 /* Fix padding length and Next Protocol in padding itself. */ 869 pad[padding - 2] = padding - 2; 870 m_copydata(m, protoff, sizeof(uint8_t), pad + padding - 1); 871 872 /* Fix Next Protocol in IPv4/IPv6 header. */ 873 prot = IPPROTO_ESP; 874 m_copyback(m, protoff, sizeof(uint8_t), &prot); 875 876 /* Get crypto descriptors. */ 877 crp = crypto_getreq(esph && espx ? 2 : 1); 878 if (crp == NULL) { 879 DPRINTF(("%s: failed to acquire crypto descriptors\n", 880 __func__)); 881 ESP_STATINC(ESP_STAT_CRYPTO); 882 error = ENOBUFS; 883 goto bad; 884 } 885 886 if (espx) { 887 crde = crp->crp_desc; 888 crda = crde->crd_next; 889 890 /* Encryption descriptor. */ 891 crde->crd_skip = skip + hlen; 892 if (espx->type == CRYPTO_AES_GMAC) 893 crde->crd_len = 0; 894 else 895 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 896 crde->crd_flags = CRD_F_ENCRYPT; 897 crde->crd_inject = skip + hlen - sav->ivlen; 898 899 /* Encryption operation. */ 900 crde->crd_alg = espx->type; 901 crde->crd_key = _KEYBUF(sav->key_enc); 902 crde->crd_klen = _KEYBITS(sav->key_enc); 903 /* XXX Rounds ? */ 904 } else 905 crda = crp->crp_desc; 906 907 /* IPsec-specific opaque crypto info. */ 908 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 909 if (tc == NULL) { 910 crypto_freereq(crp); 911 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); 912 ESP_STATINC(ESP_STAT_CRYPTO); 913 error = ENOBUFS; 914 goto bad; 915 } 916 917 { 918 int s = pserialize_read_enter(); 919 920 /* 921 * Take another reference to the SP and the SA for opencrypto callback. 922 */ 923 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD || 924 sav->state == SADB_SASTATE_DEAD)) { 925 pserialize_read_exit(s); 926 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 927 crypto_freereq(crp); 928 ESP_STATINC(ESP_STAT_NOTDB); 929 error = ENOENT; 930 goto bad; 931 } 932 KEY_SP_REF(isr->sp); 933 KEY_SA_REF(sav); 934 pserialize_read_exit(s); 935 } 936 937 /* Callback parameters */ 938 tc->tc_isr = isr; 939 tc->tc_spi = sav->spi; 940 tc->tc_dst = saidx->dst; 941 tc->tc_proto = saidx->proto; 942 tc->tc_sav = sav; 943 944 /* Crypto operation descriptor. */ 945 crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ 946 crp->crp_flags = CRYPTO_F_IMBUF; 947 crp->crp_buf = m; 948 crp->crp_callback = esp_output_cb; 949 crp->crp_opaque = tc; 950 crp->crp_sid = sav->tdb_cryptoid; 951 952 if (esph) { 953 /* Authentication descriptor. */ 954 crda->crd_skip = skip; 955 if (espx && espx->type == CRYPTO_AES_GCM_16) 956 crda->crd_len = hlen - sav->ivlen; 957 else 958 crda->crd_len = m->m_pkthdr.len - (skip + alen); 959 crda->crd_inject = m->m_pkthdr.len - alen; 960 961 /* Authentication operation. */ 962 crda->crd_alg = esph->type; 963 if (espx && (espx->type == CRYPTO_AES_GCM_16 || 964 espx->type == CRYPTO_AES_GMAC)) { 965 crda->crd_key = _KEYBUF(sav->key_enc); 966 crda->crd_klen = _KEYBITS(sav->key_enc); 967 } else { 968 crda->crd_key = _KEYBUF(sav->key_auth); 969 crda->crd_klen = _KEYBITS(sav->key_auth); 970 } 971 } 972 973 return crypto_dispatch(crp); 974 bad: 975 if (m) 976 m_freem(m); 977 return (error); 978 } 979 980 /* 981 * ESP output callback from the crypto driver. 982 */ 983 static int 984 esp_output_cb(struct cryptop *crp) 985 { 986 struct tdb_crypto *tc; 987 const struct ipsecrequest *isr; 988 struct secasvar *sav; 989 struct mbuf *m; 990 int err, error; 991 IPSEC_DECLARE_LOCK_VARIABLE; 992 993 KASSERT(crp->crp_opaque != NULL); 994 tc = crp->crp_opaque; 995 m = crp->crp_buf; 996 997 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 998 999 isr = tc->tc_isr; 1000 sav = tc->tc_sav; 1001 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD)) { 1002 ESP_STATINC(ESP_STAT_NOTDB); 1003 IPSECLOG(LOG_DEBUG, 1004 "SP is being destroyed while in crypto (id=%u)\n", 1005 isr->sp->id); 1006 error = ENOENT; 1007 goto bad; 1008 } 1009 if (__predict_false(!SADB_SASTATE_USABLE_P(sav))) { 1010 KEY_SA_UNREF(&sav); 1011 sav = KEY_LOOKUP_SA(&tc->tc_dst, tc->tc_proto, tc->tc_spi, 0, 0); 1012 if (sav == NULL) { 1013 char buf[IPSEC_ADDRSTRLEN]; 1014 ESP_STATINC(ESP_STAT_NOTDB); 1015 DPRINTF(("%s: SA expired while in crypto (SA %s/%08lx " 1016 "proto %u)\n", __func__, 1017 ipsec_address(&tc->tc_dst, buf, sizeof(buf)), 1018 (u_long) ntohl(tc->tc_spi), tc->tc_proto)); 1019 error = ENOBUFS; /*XXX*/ 1020 goto bad; 1021 } 1022 } 1023 1024 /* Check for crypto errors. */ 1025 if (crp->crp_etype) { 1026 /* Reset session ID. */ 1027 if (sav->tdb_cryptoid != 0) 1028 sav->tdb_cryptoid = crp->crp_sid; 1029 1030 if (crp->crp_etype == EAGAIN) { 1031 IPSEC_RELEASE_GLOBAL_LOCKS(); 1032 return crypto_dispatch(crp); 1033 } 1034 1035 ESP_STATINC(ESP_STAT_NOXFORM); 1036 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 1037 error = crp->crp_etype; 1038 goto bad; 1039 } 1040 1041 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 1042 if (sav->tdb_authalgxform != NULL) 1043 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 1044 1045 /* Release crypto descriptors. */ 1046 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 1047 crypto_freereq(crp); 1048 1049 #ifdef IPSEC_DEBUG 1050 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */ 1051 if (ipsec_integrity) { 1052 static unsigned char ipseczeroes[AH_ALEN_MAX]; 1053 const struct auth_hash *esph; 1054 1055 /* 1056 * Corrupt HMAC if we want to test integrity verification of 1057 * the other side. 1058 */ 1059 esph = sav->tdb_authalgxform; 1060 if (esph != NULL) { 1061 m_copyback(m, m->m_pkthdr.len - esph->authsize, 1062 esph->authsize, ipseczeroes); 1063 } 1064 } 1065 #endif 1066 1067 /* NB: m is reclaimed by ipsec_process_done. */ 1068 err = ipsec_process_done(m, isr, sav); 1069 KEY_SA_UNREF(&sav); 1070 KEY_SP_UNREF(&isr->sp); 1071 IPSEC_RELEASE_GLOBAL_LOCKS(); 1072 return err; 1073 bad: 1074 if (sav) 1075 KEY_SA_UNREF(&sav); 1076 KEY_SP_UNREF(&isr->sp); 1077 IPSEC_RELEASE_GLOBAL_LOCKS(); 1078 if (m) 1079 m_freem(m); 1080 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 1081 crypto_freereq(crp); 1082 return error; 1083 } 1084 1085 static struct xformsw esp_xformsw = { 1086 .xf_type = XF_ESP, 1087 .xf_flags = XFT_CONF|XFT_AUTH, 1088 .xf_name = "IPsec ESP", 1089 .xf_init = esp_init, 1090 .xf_zeroize = esp_zeroize, 1091 .xf_input = esp_input, 1092 .xf_output = esp_output, 1093 .xf_next = NULL, 1094 }; 1095 1096 void 1097 esp_attach(void) 1098 { 1099 1100 espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS); 1101 1102 extern int ah_max_authsize; 1103 KASSERT(ah_max_authsize != 0); 1104 esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize; 1105 esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size, 1106 coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET, 1107 NULL, NULL, NULL); 1108 1109 #define MAXIV(xform) \ 1110 if (xform.ivsize > esp_max_ivlen) \ 1111 esp_max_ivlen = xform.ivsize \ 1112 1113 esp_max_ivlen = 0; 1114 MAXIV(enc_xform_des); /* SADB_EALG_DESCBC */ 1115 MAXIV(enc_xform_3des); /* SADB_EALG_3DESCBC */ 1116 MAXIV(enc_xform_rijndael128); /* SADB_X_EALG_AES */ 1117 MAXIV(enc_xform_blf); /* SADB_X_EALG_BLOWFISHCBC */ 1118 MAXIV(enc_xform_cast5); /* SADB_X_EALG_CAST128CBC */ 1119 MAXIV(enc_xform_skipjack); /* SADB_X_EALG_SKIPJACK */ 1120 MAXIV(enc_xform_camellia); /* SADB_X_EALG_CAMELLIACBC */ 1121 MAXIV(enc_xform_aes_ctr); /* SADB_X_EALG_AESCTR */ 1122 MAXIV(enc_xform_null); /* SADB_EALG_NULL */ 1123 1124 xform_register(&esp_xformsw); 1125 #undef MAXIV 1126 } 1127