1 /* $NetBSD: xform_esp.c,v 1.85 2018/05/01 08:27:13 maxv Exp $ */ 2 /* $FreeBSD: xform_esp.c,v 1.2.2.1 2003/01/24 05:11:36 sam Exp $ */ 3 /* $OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */ 4 5 /* 6 * The authors of this code are John Ioannidis (ji@tla.org), 7 * Angelos D. Keromytis (kermit@csd.uch.gr) and 8 * Niels Provos (provos@physnet.uni-hamburg.de). 9 * 10 * The original version of this code was written by John Ioannidis 11 * for BSD/OS in Athens, Greece, in November 1995. 12 * 13 * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996, 14 * by Angelos D. Keromytis. 15 * 16 * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis 17 * and Niels Provos. 18 * 19 * Additional features in 1999 by Angelos D. Keromytis. 20 * 21 * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis, 22 * Angelos D. Keromytis and Niels Provos. 23 * Copyright (c) 2001 Angelos D. Keromytis. 24 * 25 * Permission to use, copy, and modify this software with or without fee 26 * is hereby granted, provided that this entire notice is included in 27 * all copies of any software which is or includes a copy or 28 * modification of this software. 29 * You may use this code under the GNU public license if you so wish. Please 30 * contribute changes back to the authors under this freer than GPL license 31 * so that we may further the use of strong encryption without limitations to 32 * all. 33 * 34 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 35 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 36 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 37 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 38 * PURPOSE. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: xform_esp.c,v 1.85 2018/05/01 08:27:13 maxv Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_inet.h" 46 #include "opt_ipsec.h" 47 #endif 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/mbuf.h> 52 #include <sys/socket.h> 53 #include <sys/syslog.h> 54 #include <sys/kernel.h> 55 #include <sys/sysctl.h> 56 #include <sys/cprng.h> 57 #include <sys/pool.h> 58 #include <sys/pserialize.h> 59 60 #include <net/if.h> 61 62 #include <netinet/in.h> 63 #include <netinet/in_systm.h> 64 #include <netinet/ip.h> 65 #include <netinet/ip_ecn.h> 66 #include <netinet/ip6.h> 67 68 #include <net/route.h> 69 #include <netipsec/ipsec.h> 70 #include <netipsec/ipsec_private.h> 71 #include <netipsec/ah.h> 72 #include <netipsec/ah_var.h> 73 #include <netipsec/esp.h> 74 #include <netipsec/esp_var.h> 75 #include <netipsec/xform.h> 76 77 #ifdef INET6 78 #include <netinet6/ip6_var.h> 79 #include <netipsec/ipsec6.h> 80 #endif 81 82 #include <netipsec/key.h> 83 #include <netipsec/key_debug.h> 84 85 #include <opencrypto/cryptodev.h> 86 87 percpu_t *espstat_percpu; 88 89 int esp_enable = 1; 90 91 static int esp_max_ivlen; /* max iv length over all algorithms */ 92 93 static int esp_input_cb(struct cryptop *op); 94 static int esp_output_cb(struct cryptop *crp); 95 96 const uint8_t esp_stats[256] = { SADB_EALG_STATS_INIT }; 97 98 static pool_cache_t esp_tdb_crypto_pool_cache; 99 static size_t esp_pool_item_size; 100 101 /* 102 * NB: this is public for use by the PF_KEY support. 103 * NB: if you add support here; be sure to add code to esp_attach below! 104 */ 105 const struct enc_xform * 106 esp_algorithm_lookup(int alg) 107 { 108 109 switch (alg) { 110 case SADB_EALG_DESCBC: 111 return &enc_xform_des; 112 case SADB_EALG_3DESCBC: 113 return &enc_xform_3des; 114 case SADB_X_EALG_AES: 115 return &enc_xform_rijndael128; 116 case SADB_X_EALG_BLOWFISHCBC: 117 return &enc_xform_blf; 118 case SADB_X_EALG_CAST128CBC: 119 return &enc_xform_cast5; 120 case SADB_X_EALG_SKIPJACK: 121 return &enc_xform_skipjack; 122 case SADB_X_EALG_CAMELLIACBC: 123 return &enc_xform_camellia; 124 case SADB_X_EALG_AESCTR: 125 return &enc_xform_aes_ctr; 126 case SADB_X_EALG_AESGCM16: 127 return &enc_xform_aes_gcm; 128 case SADB_X_EALG_AESGMAC: 129 return &enc_xform_aes_gmac; 130 case SADB_EALG_NULL: 131 return &enc_xform_null; 132 } 133 return NULL; 134 } 135 136 size_t 137 esp_hdrsiz(const struct secasvar *sav) 138 { 139 size_t size; 140 141 if (sav != NULL) { 142 /*XXX not right for null algorithm--does it matter??*/ 143 KASSERT(sav->tdb_encalgxform != NULL); 144 if (sav->flags & SADB_X_EXT_OLD) 145 size = sizeof(struct esp); 146 else 147 size = sizeof(struct newesp); 148 size += sav->tdb_encalgxform->ivsize + 9; 149 /*XXX need alg check???*/ 150 if (sav->tdb_authalgxform != NULL && sav->replay) 151 size += ah_hdrsiz(sav); 152 } else { 153 /* 154 * base header size 155 * + max iv length for CBC mode 156 * + max pad length 157 * + sizeof(pad length field) 158 * + sizeof(next header field) 159 * + max icv supported. 160 */ 161 size = sizeof(struct newesp) + esp_max_ivlen + 9 + 162 ah_hdrsiz(NULL); 163 } 164 return size; 165 } 166 167 /* 168 * esp_init() is called when an SPI is being set up. 169 */ 170 static int 171 esp_init(struct secasvar *sav, const struct xformsw *xsp) 172 { 173 const struct enc_xform *txform; 174 struct cryptoini cria, crie, *cr; 175 int keylen; 176 int error; 177 178 txform = esp_algorithm_lookup(sav->alg_enc); 179 if (txform == NULL) { 180 DPRINTF(("%s: unsupported encryption algorithm %d\n", __func__, 181 sav->alg_enc)); 182 return EINVAL; 183 } 184 if (sav->key_enc == NULL) { 185 DPRINTF(("%s: no encoding key for %s algorithm\n", __func__, 186 txform->name)); 187 return EINVAL; 188 } 189 if ((sav->flags&(SADB_X_EXT_OLD|SADB_X_EXT_IV4B)) == SADB_X_EXT_IV4B) { 190 DPRINTF(("%s: 4-byte IV not supported with protocol\n", 191 __func__)); 192 return EINVAL; 193 } 194 keylen = _KEYLEN(sav->key_enc); 195 if (txform->minkey > keylen || keylen > txform->maxkey) { 196 DPRINTF(("%s: invalid key length %u, must be in " 197 "the range [%u..%u] for algorithm %s\n", __func__, 198 keylen, txform->minkey, txform->maxkey, txform->name)); 199 return EINVAL; 200 } 201 202 sav->ivlen = txform->ivsize; 203 204 /* 205 * Setup AH-related state. 206 */ 207 if (sav->alg_auth != 0) { 208 error = ah_init0(sav, xsp, &cria); 209 if (error) 210 return error; 211 } 212 213 /* NB: override anything set in ah_init0 */ 214 sav->tdb_xform = xsp; 215 sav->tdb_encalgxform = txform; 216 217 switch (sav->alg_enc) { 218 case SADB_X_EALG_AESGCM16: 219 case SADB_X_EALG_AESGMAC: 220 switch (keylen) { 221 case 20: 222 sav->alg_auth = SADB_X_AALG_AES128GMAC; 223 sav->tdb_authalgxform = &auth_hash_gmac_aes_128; 224 break; 225 case 28: 226 sav->alg_auth = SADB_X_AALG_AES192GMAC; 227 sav->tdb_authalgxform = &auth_hash_gmac_aes_192; 228 break; 229 case 36: 230 sav->alg_auth = SADB_X_AALG_AES256GMAC; 231 sav->tdb_authalgxform = &auth_hash_gmac_aes_256; 232 break; 233 default: 234 DPRINTF(("%s: invalid key length %u, must be either of " 235 "20, 28 or 36\n", __func__, keylen)); 236 return EINVAL; 237 } 238 239 memset(&cria, 0, sizeof(cria)); 240 cria.cri_alg = sav->tdb_authalgxform->type; 241 cria.cri_klen = _KEYBITS(sav->key_enc); 242 cria.cri_key = _KEYBUF(sav->key_enc); 243 break; 244 default: 245 break; 246 } 247 248 /* Initialize crypto session. */ 249 memset(&crie, 0, sizeof(crie)); 250 crie.cri_alg = sav->tdb_encalgxform->type; 251 crie.cri_klen = _KEYBITS(sav->key_enc); 252 crie.cri_key = _KEYBUF(sav->key_enc); 253 /* XXX Rounds ? */ 254 255 if (sav->tdb_authalgxform && sav->tdb_encalgxform) { 256 /* init both auth & enc */ 257 crie.cri_next = &cria; 258 cr = &crie; 259 } else if (sav->tdb_encalgxform) { 260 cr = &crie; 261 } else if (sav->tdb_authalgxform) { 262 cr = &cria; 263 } else { 264 /* XXX cannot happen? */ 265 DPRINTF(("%s: no encoding OR authentication xform!\n", 266 __func__)); 267 return EINVAL; 268 } 269 270 return crypto_newsession(&sav->tdb_cryptoid, cr, crypto_support); 271 } 272 273 /* 274 * Paranoia. 275 */ 276 static int 277 esp_zeroize(struct secasvar *sav) 278 { 279 /* NB: ah_zerorize free's the crypto session state */ 280 int error = ah_zeroize(sav); 281 282 if (sav->key_enc) { 283 explicit_memset(_KEYBUF(sav->key_enc), 0, 284 _KEYLEN(sav->key_enc)); 285 } 286 sav->tdb_encalgxform = NULL; 287 sav->tdb_xform = NULL; 288 return error; 289 } 290 291 /* 292 * ESP input processing, called (eventually) through the protocol switch. 293 */ 294 static int 295 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff) 296 { 297 const struct auth_hash *esph; 298 const struct enc_xform *espx; 299 struct tdb_crypto *tc; 300 int plen, alen, hlen, error, stat = ESP_STAT_CRYPTO; 301 struct newesp *esp; 302 struct cryptodesc *crde; 303 struct cryptop *crp; 304 305 KASSERT(sav != NULL); 306 KASSERT(sav->tdb_encalgxform != NULL); 307 KASSERTMSG((skip&3) == 0 && (m->m_pkthdr.len&3) == 0, 308 "misaligned packet, skip %u pkt len %u", 309 skip, m->m_pkthdr.len); 310 311 /* XXX don't pullup, just copy header */ 312 IP6_EXTHDR_GET(esp, struct newesp *, m, skip, sizeof(struct newesp)); 313 if (esp == NULL) { 314 /* m already freed */ 315 return ENOBUFS; 316 } 317 318 esph = sav->tdb_authalgxform; 319 espx = sav->tdb_encalgxform; 320 321 /* Determine the ESP header length */ 322 if (sav->flags & SADB_X_EXT_OLD) 323 hlen = sizeof(struct esp) + sav->ivlen; 324 else 325 hlen = sizeof(struct newesp) + sav->ivlen; 326 /* Authenticator hash size */ 327 alen = esph ? esph->authsize : 0; 328 329 /* 330 * Verify payload length is multiple of encryption algorithm 331 * block size. 332 * 333 * NB: This works for the null algorithm because the blocksize 334 * is 4 and all packets must be 4-byte aligned regardless 335 * of the algorithm. 336 */ 337 plen = m->m_pkthdr.len - (skip + hlen + alen); 338 if ((plen & (espx->blocksize - 1)) || (plen <= 0)) { 339 char buf[IPSEC_ADDRSTRLEN]; 340 DPRINTF(("%s: payload of %d octets not a multiple of %d octets," 341 " SA %s/%08lx\n", __func__, plen, espx->blocksize, 342 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 343 (u_long) ntohl(sav->spi))); 344 stat = ESP_STAT_BADILEN; 345 error = EINVAL; 346 goto out; 347 } 348 349 /* 350 * Check sequence number. 351 */ 352 if (esph && sav->replay && !ipsec_chkreplay(ntohl(esp->esp_seq), sav)) { 353 char logbuf[IPSEC_LOGSASTRLEN]; 354 DPRINTF(("%s: packet replay check for %s\n", __func__, 355 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); 356 stat = ESP_STAT_REPLAY; 357 error = EACCES; 358 goto out; 359 } 360 361 /* Update the counters */ 362 ESP_STATADD(ESP_STAT_IBYTES, m->m_pkthdr.len - skip - hlen - alen); 363 364 /* Get crypto descriptors */ 365 crp = crypto_getreq(esph && espx ? 2 : 1); 366 if (crp == NULL) { 367 DPRINTF(("%s: failed to acquire crypto descriptors\n", 368 __func__)); 369 error = ENOBUFS; 370 goto out; 371 } 372 373 /* Get IPsec-specific opaque pointer */ 374 size_t extra __diagused = esph == NULL ? 0 : alen; 375 KASSERTMSG(sizeof(*tc) + extra <= esp_pool_item_size, 376 "sizeof(*tc) + extra=%zu > esp_pool_item_size=%zu\n", 377 sizeof(*tc) + extra, esp_pool_item_size); 378 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 379 if (tc == NULL) { 380 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); 381 error = ENOBUFS; 382 goto out1; 383 } 384 385 error = m_makewritable(&m, 0, m->m_pkthdr.len, M_NOWAIT); 386 if (error) { 387 DPRINTF(("%s: m_makewritable failed\n", __func__)); 388 goto out2; 389 } 390 391 if (esph) { 392 struct cryptodesc *crda; 393 394 KASSERT(crp->crp_desc != NULL); 395 crda = crp->crp_desc; 396 397 /* Authentication descriptor */ 398 crda->crd_skip = skip; 399 if (espx && espx->type == CRYPTO_AES_GCM_16) 400 crda->crd_len = hlen - sav->ivlen; 401 else 402 crda->crd_len = m->m_pkthdr.len - (skip + alen); 403 crda->crd_inject = m->m_pkthdr.len - alen; 404 405 crda->crd_alg = esph->type; 406 if (espx && (espx->type == CRYPTO_AES_GCM_16 || 407 espx->type == CRYPTO_AES_GMAC)) { 408 crda->crd_key = _KEYBUF(sav->key_enc); 409 crda->crd_klen = _KEYBITS(sav->key_enc); 410 } else { 411 crda->crd_key = _KEYBUF(sav->key_auth); 412 crda->crd_klen = _KEYBITS(sav->key_auth); 413 } 414 415 /* Copy the authenticator */ 416 m_copydata(m, m->m_pkthdr.len - alen, alen, (tc + 1)); 417 418 /* Chain authentication request */ 419 crde = crda->crd_next; 420 } else { 421 crde = crp->crp_desc; 422 } 423 424 { 425 int s = pserialize_read_enter(); 426 427 /* 428 * Take another reference to the SA for opencrypto callback. 429 */ 430 if (__predict_false(sav->state == SADB_SASTATE_DEAD)) { 431 pserialize_read_exit(s); 432 stat = ESP_STAT_NOTDB; 433 error = ENOENT; 434 goto out2; 435 } 436 KEY_SA_REF(sav); 437 pserialize_read_exit(s); 438 } 439 440 /* Crypto operation descriptor */ 441 crp->crp_ilen = m->m_pkthdr.len; /* Total input length */ 442 crp->crp_flags = CRYPTO_F_IMBUF; 443 crp->crp_buf = m; 444 crp->crp_callback = esp_input_cb; 445 crp->crp_sid = sav->tdb_cryptoid; 446 crp->crp_opaque = tc; 447 448 /* These are passed as-is to the callback */ 449 tc->tc_spi = sav->spi; 450 tc->tc_dst = sav->sah->saidx.dst; 451 tc->tc_proto = sav->sah->saidx.proto; 452 tc->tc_protoff = protoff; 453 tc->tc_skip = skip; 454 tc->tc_sav = sav; 455 456 /* Decryption descriptor */ 457 if (espx) { 458 KASSERTMSG(crde != NULL, "null esp crypto descriptor"); 459 crde->crd_skip = skip + hlen; 460 if (espx->type == CRYPTO_AES_GMAC) 461 crde->crd_len = 0; 462 else 463 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 464 crde->crd_inject = skip + hlen - sav->ivlen; 465 466 crde->crd_alg = espx->type; 467 crde->crd_key = _KEYBUF(sav->key_enc); 468 crde->crd_klen = _KEYBITS(sav->key_enc); 469 /* XXX Rounds ? */ 470 } 471 472 return crypto_dispatch(crp); 473 474 out2: 475 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 476 out1: 477 crypto_freereq(crp); 478 out: 479 ESP_STATINC(stat); 480 m_freem(m); 481 return error; 482 } 483 484 #ifdef INET6 485 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) do { \ 486 if (saidx->dst.sa.sa_family == AF_INET6) { \ 487 error = ipsec6_common_input_cb(m, sav, skip, protoff); \ 488 } else { \ 489 error = ipsec4_common_input_cb(m, sav, skip, protoff); \ 490 } \ 491 } while (0) 492 #else 493 #define IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff) \ 494 (error = ipsec4_common_input_cb(m, sav, skip, protoff)) 495 #endif 496 497 /* 498 * ESP input callback from the crypto driver. 499 */ 500 static int 501 esp_input_cb(struct cryptop *crp) 502 { 503 char buf[IPSEC_ADDRSTRLEN]; 504 uint8_t lastthree[3], aalg[AH_ALEN_MAX]; 505 int hlen, skip, protoff, error; 506 struct mbuf *m; 507 const struct auth_hash *esph; 508 struct tdb_crypto *tc; 509 struct secasvar *sav; 510 struct secasindex *saidx; 511 void *ptr; 512 uint16_t dport; 513 uint16_t sport; 514 IPSEC_DECLARE_LOCK_VARIABLE; 515 516 KASSERT(crp->crp_desc != NULL); 517 KASSERT(crp->crp_opaque != NULL); 518 519 tc = crp->crp_opaque; 520 skip = tc->tc_skip; 521 protoff = tc->tc_protoff; 522 m = crp->crp_buf; 523 524 /* find the source port for NAT-T */ 525 nat_t_ports_get(m, &dport, &sport); 526 527 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 528 529 sav = tc->tc_sav; 530 saidx = &sav->sah->saidx; 531 KASSERTMSG(saidx->dst.sa.sa_family == AF_INET || 532 saidx->dst.sa.sa_family == AF_INET6, 533 "unexpected protocol family %u", saidx->dst.sa.sa_family); 534 535 esph = sav->tdb_authalgxform; 536 537 /* Check for crypto errors */ 538 if (crp->crp_etype) { 539 /* Reset the session ID */ 540 if (sav->tdb_cryptoid != 0) 541 sav->tdb_cryptoid = crp->crp_sid; 542 543 if (crp->crp_etype == EAGAIN) { 544 KEY_SA_UNREF(&sav); 545 IPSEC_RELEASE_GLOBAL_LOCKS(); 546 return crypto_dispatch(crp); 547 } 548 549 ESP_STATINC(ESP_STAT_NOXFORM); 550 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 551 error = crp->crp_etype; 552 goto bad; 553 } 554 555 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 556 557 /* If authentication was performed, check now. */ 558 if (esph != NULL) { 559 /* 560 * If we have a tag, it means an IPsec-aware NIC did 561 * the verification for us. Otherwise we need to 562 * check the authentication calculation. 563 */ 564 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 565 /* Copy the authenticator from the packet */ 566 m_copydata(m, m->m_pkthdr.len - esph->authsize, 567 esph->authsize, aalg); 568 569 ptr = (tc + 1); 570 571 /* Verify authenticator */ 572 if (!consttime_memequal(ptr, aalg, esph->authsize)) { 573 DPRINTF(("%s: authentication hash mismatch " 574 "for packet in SA %s/%08lx\n", __func__, 575 ipsec_address(&saidx->dst, buf, 576 sizeof(buf)), (u_long) ntohl(sav->spi))); 577 ESP_STATINC(ESP_STAT_BADAUTH); 578 error = EACCES; 579 goto bad; 580 } 581 582 /* Remove trailing authenticator */ 583 m_adj(m, -(esph->authsize)); 584 } 585 586 /* Release the crypto descriptors */ 587 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 588 tc = NULL; 589 crypto_freereq(crp); 590 crp = NULL; 591 592 /* 593 * Packet is now decrypted. 594 */ 595 m->m_flags |= M_DECRYPTED; 596 597 /* 598 * Update replay sequence number, if appropriate. 599 */ 600 if (sav->replay) { 601 uint32_t seq; 602 603 m_copydata(m, skip + offsetof(struct newesp, esp_seq), 604 sizeof(seq), &seq); 605 if (ipsec_updatereplay(ntohl(seq), sav)) { 606 char logbuf[IPSEC_LOGSASTRLEN]; 607 DPRINTF(("%s: packet replay check for %s\n", __func__, 608 ipsec_logsastr(sav, logbuf, sizeof(logbuf)))); 609 ESP_STATINC(ESP_STAT_REPLAY); 610 error = ENOBUFS; 611 goto bad; 612 } 613 } 614 615 /* Determine the ESP header length */ 616 if (sav->flags & SADB_X_EXT_OLD) 617 hlen = sizeof(struct esp) + sav->ivlen; 618 else 619 hlen = sizeof(struct newesp) + sav->ivlen; 620 621 /* Remove the ESP header and IV from the mbuf. */ 622 error = m_striphdr(m, skip, hlen); 623 if (error) { 624 ESP_STATINC(ESP_STAT_HDROPS); 625 DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__, 626 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 627 (u_long) ntohl(sav->spi))); 628 goto bad; 629 } 630 631 /* Save the last three bytes of decrypted data */ 632 m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree); 633 634 /* Verify pad length */ 635 if (lastthree[1] + 2 > m->m_pkthdr.len - skip) { 636 ESP_STATINC(ESP_STAT_BADILEN); 637 DPRINTF(("%s: invalid padding length %d " 638 "for %u byte packet in SA %s/%08lx\n", __func__, 639 lastthree[1], m->m_pkthdr.len - skip, 640 ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)), 641 (u_long) ntohl(sav->spi))); 642 error = EINVAL; 643 goto bad; 644 } 645 646 /* Verify correct decryption by checking the last padding bytes */ 647 if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) { 648 if (lastthree[1] != lastthree[0] && lastthree[1] != 0) { 649 ESP_STATINC(ESP_STAT_BADENC); 650 DPRINTF(("%s: decryption failed for packet in SA " 651 "%s/%08lx\n", __func__, 652 ipsec_address(&sav->sah->saidx.dst, buf, 653 sizeof(buf)), (u_long) ntohl(sav->spi))); 654 DPRINTF(("%s: %x %x\n", __func__, lastthree[0], 655 lastthree[1])); 656 error = EINVAL; 657 goto bad; 658 } 659 } 660 661 /* Trim the mbuf chain to remove trailing authenticator and padding */ 662 m_adj(m, -(lastthree[1] + 2)); 663 664 /* Restore the Next Protocol field */ 665 m_copyback(m, protoff, sizeof(uint8_t), lastthree + 2); 666 667 IPSEC_COMMON_INPUT_CB(m, sav, skip, protoff); 668 669 KEY_SA_UNREF(&sav); 670 IPSEC_RELEASE_GLOBAL_LOCKS(); 671 return error; 672 bad: 673 if (sav) 674 KEY_SA_UNREF(&sav); 675 IPSEC_RELEASE_GLOBAL_LOCKS(); 676 if (m != NULL) 677 m_freem(m); 678 if (tc != NULL) 679 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 680 if (crp != NULL) 681 crypto_freereq(crp); 682 return error; 683 } 684 685 /* 686 * ESP output routine, called by ipsec[46]_process_packet(). 687 */ 688 static int 689 esp_output(struct mbuf *m, const struct ipsecrequest *isr, struct secasvar *sav, 690 struct mbuf **mp, int skip, int protoff) 691 { 692 char buf[IPSEC_ADDRSTRLEN]; 693 const struct enc_xform *espx; 694 const struct auth_hash *esph; 695 int hlen, rlen, padding, blks, alen, i, roff; 696 struct mbuf *mo = NULL; 697 struct tdb_crypto *tc; 698 struct secasindex *saidx; 699 unsigned char *pad; 700 uint8_t prot; 701 int error, maxpacketsize; 702 703 struct cryptodesc *crde = NULL, *crda = NULL; 704 struct cryptop *crp; 705 706 esph = sav->tdb_authalgxform; 707 KASSERT(sav->tdb_encalgxform != NULL); 708 espx = sav->tdb_encalgxform; 709 710 if (sav->flags & SADB_X_EXT_OLD) 711 hlen = sizeof(struct esp) + sav->ivlen; 712 else 713 hlen = sizeof(struct newesp) + sav->ivlen; 714 715 rlen = m->m_pkthdr.len - skip; /* Raw payload length. */ 716 /* 717 * NB: The null encoding transform has a blocksize of 4 718 * so that headers are properly aligned. 719 */ 720 blks = espx->blocksize; /* IV blocksize */ 721 722 /* XXX clamp padding length a la KAME??? */ 723 padding = ((blks - ((rlen + 2) % blks)) % blks) + 2; 724 725 if (esph) 726 alen = esph->authsize; 727 else 728 alen = 0; 729 730 ESP_STATINC(ESP_STAT_OUTPUT); 731 732 saidx = &sav->sah->saidx; 733 /* Check for maximum packet size violations. */ 734 switch (saidx->dst.sa.sa_family) { 735 #ifdef INET 736 case AF_INET: 737 maxpacketsize = IP_MAXPACKET; 738 break; 739 #endif 740 #ifdef INET6 741 case AF_INET6: 742 maxpacketsize = IPV6_MAXPACKET; 743 break; 744 #endif 745 default: 746 DPRINTF(("%s: unknown/unsupported protocol family %d, " 747 "SA %s/%08lx\n", __func__, saidx->dst.sa.sa_family, 748 ipsec_address(&saidx->dst, buf, sizeof(buf)), 749 (u_long)ntohl(sav->spi))); 750 ESP_STATINC(ESP_STAT_NOPF); 751 error = EPFNOSUPPORT; 752 goto bad; 753 } 754 if (skip + hlen + rlen + padding + alen > maxpacketsize) { 755 DPRINTF(("%s: packet in SA %s/%08lx got too big (len %u, " 756 "max len %u)\n", __func__, 757 ipsec_address(&saidx->dst, buf, sizeof(buf)), 758 (u_long) ntohl(sav->spi), 759 skip + hlen + rlen + padding + alen, maxpacketsize)); 760 ESP_STATINC(ESP_STAT_TOOBIG); 761 error = EMSGSIZE; 762 goto bad; 763 } 764 765 /* Update the counters. */ 766 ESP_STATADD(ESP_STAT_OBYTES, m->m_pkthdr.len - skip); 767 768 m = m_clone(m); 769 if (m == NULL) { 770 DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__, 771 ipsec_address(&saidx->dst, buf, sizeof(buf)), 772 (u_long) ntohl(sav->spi))); 773 ESP_STATINC(ESP_STAT_HDROPS); 774 error = ENOBUFS; 775 goto bad; 776 } 777 778 /* Inject ESP header. */ 779 mo = m_makespace(m, skip, hlen, &roff); 780 if (mo == NULL) { 781 DPRINTF(("%s: failed to inject %u byte ESP hdr for SA " 782 "%s/%08lx\n", __func__, hlen, 783 ipsec_address(&saidx->dst, buf, sizeof(buf)), 784 (u_long) ntohl(sav->spi))); 785 ESP_STATINC(ESP_STAT_HDROPS); 786 error = ENOBUFS; 787 goto bad; 788 } 789 790 /* Initialize ESP header. */ 791 memcpy(mtod(mo, char *) + roff, &sav->spi, sizeof(uint32_t)); 792 if (sav->replay) { 793 uint32_t replay; 794 795 #ifdef IPSEC_DEBUG 796 /* Emulate replay attack when ipsec_replay is TRUE. */ 797 if (!ipsec_replay) 798 #endif 799 sav->replay->count++; 800 801 replay = htonl(sav->replay->count); 802 memcpy(mtod(mo,char *) + roff + sizeof(uint32_t), &replay, 803 sizeof(uint32_t)); 804 } 805 806 /* 807 * Add padding -- better to do it ourselves than use the crypto engine, 808 * although if/when we support compression, we'd have to do that. 809 */ 810 pad = m_pad(m, padding + alen); 811 if (pad == NULL) { 812 DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__, 813 ipsec_address(&saidx->dst, buf, sizeof(buf)), 814 (u_long) ntohl(sav->spi))); 815 m = NULL; /* NB: free'd by m_pad */ 816 error = ENOBUFS; 817 goto bad; 818 } 819 820 /* 821 * Add padding: random, zero, or self-describing. 822 */ 823 switch (sav->flags & SADB_X_EXT_PMASK) { 824 case SADB_X_EXT_PSEQ: 825 for (i = 0; i < padding - 2; i++) 826 pad[i] = i+1; 827 break; 828 case SADB_X_EXT_PRAND: 829 (void)cprng_fast(pad, padding - 2); 830 break; 831 case SADB_X_EXT_PZERO: 832 default: 833 memset(pad, 0, padding - 2); 834 break; 835 } 836 837 /* Fix padding length and Next Protocol in padding itself. */ 838 pad[padding - 2] = padding - 2; 839 m_copydata(m, protoff, sizeof(uint8_t), pad + padding - 1); 840 841 /* Fix Next Protocol in IPv4/IPv6 header. */ 842 prot = IPPROTO_ESP; 843 m_copyback(m, protoff, sizeof(uint8_t), &prot); 844 845 /* Get crypto descriptors. */ 846 crp = crypto_getreq(esph && espx ? 2 : 1); 847 if (crp == NULL) { 848 DPRINTF(("%s: failed to acquire crypto descriptors\n", 849 __func__)); 850 ESP_STATINC(ESP_STAT_CRYPTO); 851 error = ENOBUFS; 852 goto bad; 853 } 854 855 if (espx) { 856 crde = crp->crp_desc; 857 crda = crde->crd_next; 858 859 /* Encryption descriptor. */ 860 crde->crd_skip = skip + hlen; 861 if (espx->type == CRYPTO_AES_GMAC) 862 crde->crd_len = 0; 863 else 864 crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen); 865 crde->crd_flags = CRD_F_ENCRYPT; 866 crde->crd_inject = skip + hlen - sav->ivlen; 867 868 /* Encryption operation. */ 869 crde->crd_alg = espx->type; 870 crde->crd_key = _KEYBUF(sav->key_enc); 871 crde->crd_klen = _KEYBITS(sav->key_enc); 872 /* XXX Rounds ? */ 873 } else 874 crda = crp->crp_desc; 875 876 /* IPsec-specific opaque crypto info. */ 877 tc = pool_cache_get(esp_tdb_crypto_pool_cache, PR_NOWAIT); 878 if (tc == NULL) { 879 crypto_freereq(crp); 880 DPRINTF(("%s: failed to allocate tdb_crypto\n", __func__)); 881 ESP_STATINC(ESP_STAT_CRYPTO); 882 error = ENOBUFS; 883 goto bad; 884 } 885 886 { 887 int s = pserialize_read_enter(); 888 889 /* 890 * Take another reference to the SP and the SA for opencrypto callback. 891 */ 892 if (__predict_false(isr->sp->state == IPSEC_SPSTATE_DEAD || 893 sav->state == SADB_SASTATE_DEAD)) { 894 pserialize_read_exit(s); 895 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 896 crypto_freereq(crp); 897 ESP_STATINC(ESP_STAT_NOTDB); 898 error = ENOENT; 899 goto bad; 900 } 901 KEY_SP_REF(isr->sp); 902 KEY_SA_REF(sav); 903 pserialize_read_exit(s); 904 } 905 906 /* Callback parameters */ 907 tc->tc_isr = isr; 908 tc->tc_spi = sav->spi; 909 tc->tc_dst = saidx->dst; 910 tc->tc_proto = saidx->proto; 911 tc->tc_sav = sav; 912 913 /* Crypto operation descriptor. */ 914 crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */ 915 crp->crp_flags = CRYPTO_F_IMBUF; 916 crp->crp_buf = m; 917 crp->crp_callback = esp_output_cb; 918 crp->crp_opaque = tc; 919 crp->crp_sid = sav->tdb_cryptoid; 920 921 if (esph) { 922 /* Authentication descriptor. */ 923 crda->crd_skip = skip; 924 if (espx && espx->type == CRYPTO_AES_GCM_16) 925 crda->crd_len = hlen - sav->ivlen; 926 else 927 crda->crd_len = m->m_pkthdr.len - (skip + alen); 928 crda->crd_inject = m->m_pkthdr.len - alen; 929 930 /* Authentication operation. */ 931 crda->crd_alg = esph->type; 932 if (espx && (espx->type == CRYPTO_AES_GCM_16 || 933 espx->type == CRYPTO_AES_GMAC)) { 934 crda->crd_key = _KEYBUF(sav->key_enc); 935 crda->crd_klen = _KEYBITS(sav->key_enc); 936 } else { 937 crda->crd_key = _KEYBUF(sav->key_auth); 938 crda->crd_klen = _KEYBITS(sav->key_auth); 939 } 940 } 941 942 return crypto_dispatch(crp); 943 944 bad: 945 if (m) 946 m_freem(m); 947 return error; 948 } 949 950 /* 951 * ESP output callback from the crypto driver. 952 */ 953 static int 954 esp_output_cb(struct cryptop *crp) 955 { 956 struct tdb_crypto *tc; 957 const struct ipsecrequest *isr; 958 struct secasvar *sav; 959 struct mbuf *m; 960 int err, error; 961 IPSEC_DECLARE_LOCK_VARIABLE; 962 963 KASSERT(crp->crp_opaque != NULL); 964 tc = crp->crp_opaque; 965 m = crp->crp_buf; 966 967 IPSEC_ACQUIRE_GLOBAL_LOCKS(); 968 969 isr = tc->tc_isr; 970 sav = tc->tc_sav; 971 972 /* Check for crypto errors. */ 973 if (crp->crp_etype) { 974 /* Reset session ID. */ 975 if (sav->tdb_cryptoid != 0) 976 sav->tdb_cryptoid = crp->crp_sid; 977 978 if (crp->crp_etype == EAGAIN) { 979 IPSEC_RELEASE_GLOBAL_LOCKS(); 980 return crypto_dispatch(crp); 981 } 982 983 ESP_STATINC(ESP_STAT_NOXFORM); 984 DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype)); 985 error = crp->crp_etype; 986 goto bad; 987 } 988 989 ESP_STATINC(ESP_STAT_HIST + esp_stats[sav->alg_enc]); 990 if (sav->tdb_authalgxform != NULL) 991 AH_STATINC(AH_STAT_HIST + ah_stats[sav->alg_auth]); 992 993 /* Release crypto descriptors. */ 994 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 995 crypto_freereq(crp); 996 997 #ifdef IPSEC_DEBUG 998 /* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */ 999 if (ipsec_integrity) { 1000 static unsigned char ipseczeroes[AH_ALEN_MAX]; 1001 const struct auth_hash *esph; 1002 1003 /* 1004 * Corrupt HMAC if we want to test integrity verification of 1005 * the other side. 1006 */ 1007 esph = sav->tdb_authalgxform; 1008 if (esph != NULL) { 1009 m_copyback(m, m->m_pkthdr.len - esph->authsize, 1010 esph->authsize, ipseczeroes); 1011 } 1012 } 1013 #endif 1014 1015 /* NB: m is reclaimed by ipsec_process_done. */ 1016 err = ipsec_process_done(m, isr, sav); 1017 KEY_SA_UNREF(&sav); 1018 KEY_SP_UNREF(&isr->sp); 1019 IPSEC_RELEASE_GLOBAL_LOCKS(); 1020 return err; 1021 1022 bad: 1023 if (sav) 1024 KEY_SA_UNREF(&sav); 1025 KEY_SP_UNREF(&isr->sp); 1026 IPSEC_RELEASE_GLOBAL_LOCKS(); 1027 if (m) 1028 m_freem(m); 1029 pool_cache_put(esp_tdb_crypto_pool_cache, tc); 1030 crypto_freereq(crp); 1031 return error; 1032 } 1033 1034 static struct xformsw esp_xformsw = { 1035 .xf_type = XF_ESP, 1036 .xf_flags = XFT_CONF|XFT_AUTH, 1037 .xf_name = "IPsec ESP", 1038 .xf_init = esp_init, 1039 .xf_zeroize = esp_zeroize, 1040 .xf_input = esp_input, 1041 .xf_output = esp_output, 1042 .xf_next = NULL, 1043 }; 1044 1045 void 1046 esp_attach(void) 1047 { 1048 1049 espstat_percpu = percpu_alloc(sizeof(uint64_t) * ESP_NSTATS); 1050 1051 extern int ah_max_authsize; 1052 KASSERT(ah_max_authsize != 0); 1053 esp_pool_item_size = sizeof(struct tdb_crypto) + ah_max_authsize; 1054 esp_tdb_crypto_pool_cache = pool_cache_init(esp_pool_item_size, 1055 coherency_unit, 0, 0, "esp_tdb_crypto", NULL, IPL_SOFTNET, 1056 NULL, NULL, NULL); 1057 1058 #define MAXIV(xform) \ 1059 if (xform.ivsize > esp_max_ivlen) \ 1060 esp_max_ivlen = xform.ivsize \ 1061 1062 esp_max_ivlen = 0; 1063 MAXIV(enc_xform_des); /* SADB_EALG_DESCBC */ 1064 MAXIV(enc_xform_3des); /* SADB_EALG_3DESCBC */ 1065 MAXIV(enc_xform_rijndael128); /* SADB_X_EALG_AES */ 1066 MAXIV(enc_xform_blf); /* SADB_X_EALG_BLOWFISHCBC */ 1067 MAXIV(enc_xform_cast5); /* SADB_X_EALG_CAST128CBC */ 1068 MAXIV(enc_xform_skipjack); /* SADB_X_EALG_SKIPJACK */ 1069 MAXIV(enc_xform_camellia); /* SADB_X_EALG_CAMELLIACBC */ 1070 MAXIV(enc_xform_aes_ctr); /* SADB_X_EALG_AESCTR */ 1071 MAXIV(enc_xform_null); /* SADB_EALG_NULL */ 1072 1073 xform_register(&esp_xformsw); 1074 #undef MAXIV 1075 } 1076