1 /*- 2 * SPDX-License-Identifier: ISC 3 * 4 * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 5 * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net> 6 * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate) 7 * Copyright (c) 2021 Kyle Evans <kevans@FreeBSD.org> 8 * Copyright (c) 2022 The FreeBSD Foundation 9 * Copyright (c) 2023-2024 Aaron LI <aly@aaronly.me> 10 * 11 * Permission to use, copy, modify, and distribute this software for any 12 * purpose with or without fee is hereby granted, provided that the above 13 * copyright notice and this permission notice appear in all copies. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 16 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 17 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 18 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 20 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 21 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 22 */ 23 24 #include "opt_inet6.h" 25 26 #include <sys/param.h> 27 #include <sys/systm.h> 28 #include <sys/callout.h> 29 #include <sys/caps.h> 30 #include <sys/endian.h> 31 #include <sys/kernel.h> 32 #include <sys/lock.h> 33 #include <sys/malloc.h> 34 #include <sys/mbuf.h> 35 #include <sys/module.h> 36 #include <sys/objcache.h> 37 #include <sys/queue.h> 38 #include <sys/socket.h> 39 #include <sys/socketops.h> /* so_pru_*() functions */ 40 #include <sys/socketvar.h> 41 #include <sys/sockio.h> /* SIOC* ioctl commands */ 42 #include <sys/taskqueue.h> 43 #include <sys/time.h> 44 45 #include <machine/atomic.h> 46 47 #include <net/bpf.h> 48 #include <net/ethernet.h> /* ETHERMTU */ 49 #include <net/if.h> 50 #include <net/if_clone.h> 51 #include <net/if_types.h> /* IFT_WIREGUARD */ 52 #include <net/if_var.h> 53 #include <net/ifq_var.h> 54 #include <net/netisr.h> 55 #include <net/radix.h> 56 #include <net/route.h> /* struct rtentry */ 57 58 #include <netinet/in.h> 59 #include <netinet/ip.h> 60 #include <netinet/ip_icmp.h> 61 #include <netinet/ip6.h> 62 #include <netinet/icmp6.h> 63 #include <netinet6/in6_var.h> /* in6_mask2len() */ 64 65 #include "wg_cookie.h" 66 #include "wg_noise.h" 67 #include "if_wg.h" 68 69 CTASSERT(WG_KEY_SIZE >= NOISE_PUBLIC_KEY_LEN); 70 CTASSERT(WG_KEY_SIZE >= NOISE_SYMMETRIC_KEY_LEN); 71 72 #define DEFAULT_MTU (ETHERMTU - 80) 73 #define MAX_MTU (IF_MAXMTU - 80) 74 75 #ifndef ENOKEY 76 #define ENOKEY ENOENT 77 #endif 78 79 /* 80 * mbuf flags to clear after in-place encryption/decryption, so that the 81 * mbuf can be reused for re-entering the network stack or delivering to 82 * the remote peer. 83 * 84 * For example, the M_HASH and M_LENCHECKED flags must be cleared for an 85 * inbound packet; otherwise, panic is to be expected. 86 */ 87 #define MBUF_CLEARFLAGS (M_COPYFLAGS & ~(M_PKTHDR | M_EOR | M_PRIO)) 88 89 #define MAX_LOOPS 8 /* 0 means no loop allowed */ 90 #define MAX_STAGED_PKT 128 91 #define MAX_QUEUED_PKT 1024 92 #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1) 93 #define MAX_QUEUED_HANDSHAKES 4096 94 95 #define REKEY_TIMEOUT_JITTER (karc4random() % 334) /* msec */ 96 #define MAX_TIMER_HANDSHAKES (90 / REKEY_TIMEOUT) 97 #define NEW_HANDSHAKE_TIMEOUT (REKEY_TIMEOUT + KEEPALIVE_TIMEOUT) 98 #define UNDERLOAD_TIMEOUT 1 99 100 /* First byte indicating packet type on the wire */ 101 #define WG_PKT_INITIATION htole32(1) 102 #define WG_PKT_RESPONSE htole32(2) 103 #define WG_PKT_COOKIE htole32(3) 104 #define WG_PKT_DATA htole32(4) 105 106 #define WG_PKT_PADDING 16 107 #define WG_PKT_WITH_PADDING(n) \ 108 (((n) + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1)) 109 #define WG_PKT_ENCRYPTED_LEN(n) \ 110 (offsetof(struct wg_pkt_data, buf[(n)]) + NOISE_AUTHTAG_LEN) 111 #define WG_PKT_IS_INITIATION(m) \ 112 (*mtod((m), uint32_t *) == WG_PKT_INITIATION && \ 113 (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_initiation)) 114 #define WG_PKT_IS_RESPONSE(m) \ 115 (*mtod((m), uint32_t *) == WG_PKT_RESPONSE && \ 116 (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_response)) 117 #define WG_PKT_IS_COOKIE(m) \ 118 (*mtod((m), uint32_t *) == WG_PKT_COOKIE && \ 119 (size_t)(m)->m_pkthdr.len == sizeof(struct wg_pkt_cookie)) 120 #define WG_PKT_IS_DATA(m) \ 121 (*mtod((m), uint32_t *) == WG_PKT_DATA && \ 122 (size_t)(m)->m_pkthdr.len >= WG_PKT_ENCRYPTED_LEN(0)) 123 124 125 #define DPRINTF(sc, fmt, ...) \ 126 if (sc->sc_ifp->if_flags & IFF_DEBUG) \ 127 if_printf(sc->sc_ifp, fmt, ##__VA_ARGS__) 128 129 130 struct wg_pkt_initiation { 131 uint32_t t; 132 uint32_t s_idx; 133 uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 134 uint8_t es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN]; 135 uint8_t ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN]; 136 struct cookie_macs m; 137 }; 138 139 struct wg_pkt_response { 140 uint32_t t; 141 uint32_t s_idx; 142 uint32_t r_idx; 143 uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 144 uint8_t en[0 + NOISE_AUTHTAG_LEN]; 145 struct cookie_macs m; 146 }; 147 148 struct wg_pkt_cookie { 149 uint32_t t; 150 uint32_t r_idx; 151 uint8_t nonce[COOKIE_NONCE_SIZE]; 152 uint8_t ec[COOKIE_ENCRYPTED_SIZE]; 153 }; 154 155 struct wg_pkt_data { 156 uint32_t t; 157 uint32_t r_idx; 158 uint64_t counter; 159 uint8_t buf[]; 160 }; 161 162 struct wg_endpoint { 163 union { 164 struct sockaddr r_sa; 165 struct sockaddr_in r_sin; 166 #ifdef INET6 167 struct sockaddr_in6 r_sin6; 168 #endif 169 } e_remote; 170 /* 171 * NOTE: No 'e_local' on DragonFly, because the socket upcall 172 * and so_pru_soreceive() cannot provide the local 173 * (i.e., destination) address of a received packet. 174 */ 175 }; 176 177 struct aip_addr { 178 uint8_t length; /* required by the radix code */ 179 union { 180 uint8_t bytes[16]; 181 uint32_t ip; 182 uint32_t ip6[4]; 183 struct in_addr in; 184 struct in6_addr in6; 185 }; 186 }; 187 188 struct wg_aip { 189 struct radix_node a_nodes[2]; /* make the first for casting */ 190 LIST_ENTRY(wg_aip) a_entry; 191 struct aip_addr a_addr; 192 struct aip_addr a_mask; 193 struct wg_peer *a_peer; 194 sa_family_t a_af; 195 }; 196 197 enum wg_packet_state { 198 WG_PACKET_DEAD, /* to be dropped */ 199 WG_PACKET_UNCRYPTED, /* before encryption/decryption */ 200 WG_PACKET_CRYPTED, /* after encryption/decryption */ 201 }; 202 203 struct wg_packet { 204 STAILQ_ENTRY(wg_packet) p_serial; 205 STAILQ_ENTRY(wg_packet) p_parallel; 206 struct wg_endpoint p_endpoint; 207 struct noise_keypair *p_keypair; 208 uint64_t p_counter; 209 struct mbuf *p_mbuf; 210 int p_mtu; 211 sa_family_t p_af; 212 unsigned int p_state; /* atomic */ 213 }; 214 215 STAILQ_HEAD(wg_packet_list, wg_packet); 216 217 struct wg_queue { 218 struct lock q_mtx; 219 struct wg_packet_list q_queue; 220 size_t q_len; 221 }; 222 223 struct wg_peer { 224 TAILQ_ENTRY(wg_peer) p_entry; 225 unsigned long p_id; 226 struct wg_softc *p_sc; 227 228 char p_description[WG_PEER_DESCR_SIZE]; 229 230 struct noise_remote *p_remote; 231 struct cookie_maker *p_cookie; 232 233 struct lock p_endpoint_lock; 234 struct wg_endpoint p_endpoint; 235 236 struct wg_queue p_stage_queue; 237 struct wg_queue p_encrypt_serial; 238 struct wg_queue p_decrypt_serial; 239 240 bool p_enabled; 241 bool p_need_another_keepalive; 242 uint16_t p_persistent_keepalive_interval; 243 struct callout p_new_handshake; 244 struct callout p_send_keepalive; 245 struct callout p_retry_handshake; 246 struct callout p_zero_key_material; 247 struct callout p_persistent_keepalive; 248 249 struct lock p_handshake_mtx; 250 struct timespec p_handshake_complete; /* nanotime */ 251 int p_handshake_retries; 252 253 struct task p_send_task; 254 struct task p_recv_task; 255 struct taskqueue *p_send_taskqueue; 256 struct taskqueue *p_recv_taskqueue; 257 258 uint64_t *p_tx_bytes; 259 uint64_t *p_rx_bytes; 260 261 LIST_HEAD(, wg_aip) p_aips; 262 size_t p_aips_num; 263 }; 264 265 struct wg_socket { 266 struct lock so_lock; 267 struct socket *so_so4; 268 struct socket *so_so6; 269 uint32_t so_user_cookie; 270 in_port_t so_port; 271 }; 272 273 struct wg_softc { 274 LIST_ENTRY(wg_softc) sc_entry; 275 struct ifnet *sc_ifp; 276 int sc_flags; 277 278 struct wg_socket sc_socket; 279 280 TAILQ_HEAD(, wg_peer) sc_peers; 281 size_t sc_peers_num; 282 283 struct noise_local *sc_local; 284 struct cookie_checker *sc_cookie; 285 286 struct lock sc_aip_lock; 287 struct radix_node_head *sc_aip4; 288 struct radix_node_head *sc_aip6; 289 290 struct taskqueue *sc_handshake_taskqueue; 291 struct task sc_handshake_task; 292 struct wg_queue sc_handshake_queue; 293 294 struct task *sc_encrypt_tasks; /* one per CPU */ 295 struct task *sc_decrypt_tasks; /* one per CPU */ 296 struct wg_queue sc_encrypt_parallel; 297 struct wg_queue sc_decrypt_parallel; 298 int sc_encrypt_last_cpu; 299 int sc_decrypt_last_cpu; 300 301 struct lock sc_lock; 302 }; 303 304 305 static MALLOC_DEFINE(M_WG, "WG", "wireguard"); 306 static MALLOC_DEFINE(M_WG_PACKET, "WG packet", "wireguard packet"); 307 308 static const char wgname[] = "wg"; 309 310 static struct objcache *wg_packet_zone; 311 static struct lock wg_mtx; 312 static struct taskqueue **wg_taskqueues; /* one taskqueue per CPU */ 313 static struct radix_node_head *wg_maskhead; /* shared by all interfaces */ 314 static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list); 315 316 317 /* Timers */ 318 static void wg_timers_enable(struct wg_peer *); 319 static void wg_timers_disable(struct wg_peer *); 320 321 /* Allowed IP */ 322 static int wg_aip_add(struct wg_softc *, struct wg_peer *, sa_family_t, 323 const void *, uint8_t); 324 static struct wg_peer * 325 wg_aip_lookup(struct wg_softc *, sa_family_t, const void *); 326 static void wg_aip_remove_all(struct wg_softc *, struct wg_peer *); 327 328 /* Handshake */ 329 static void wg_send_initiation(struct wg_peer *); 330 static void wg_send_response(struct wg_peer *); 331 static void wg_send_cookie(struct wg_softc *, struct cookie_macs *, 332 uint32_t, struct wg_endpoint *); 333 static void wg_send_keepalive(struct wg_peer *); 334 335 /* Transport Packet Functions */ 336 static void wg_peer_send_staged(struct wg_peer *); 337 static void wg_deliver_out(void *, int); 338 static void wg_deliver_in(void *, int); 339 static void wg_upcall(struct socket *, void *, int); 340 341 /*----------------------------------------------------------------------------*/ 342 /* Packet */ 343 344 static struct wg_packet * 345 wg_packet_alloc(struct mbuf *m) 346 { 347 struct wg_packet *pkt; 348 349 if ((pkt = objcache_get(wg_packet_zone, M_NOWAIT)) == NULL) 350 return (NULL); 351 352 bzero(pkt, sizeof(*pkt)); /* objcache_get() doesn't ensure M_ZERO. */ 353 pkt->p_mbuf = m; 354 355 return (pkt); 356 } 357 358 static void 359 wg_packet_free(struct wg_packet *pkt) 360 { 361 if (pkt->p_keypair != NULL) 362 noise_keypair_put(pkt->p_keypair); 363 if (pkt->p_mbuf != NULL) 364 m_freem(pkt->p_mbuf); 365 objcache_put(wg_packet_zone, pkt); 366 } 367 368 /*----------------------------------------------------------------------------*/ 369 /* 370 * Packet Queue Functions 371 * 372 * WireGuard uses the following queues: 373 * - per-interface handshake queue: track incoming handshake packets 374 * - per-peer staged queue: track the outgoing packets sent by that peer 375 * - per-interface parallel encrypt and decrypt queues 376 * - per-peer serial encrypt and decrypt queues 377 * 378 * For one interface, the handshake packets are only tracked in the handshake 379 * queue and are processed in serial. However, all data packets are tracked 380 * in two queues: a serial queue and a parallel queue. Specifically, the 381 * outgoing packets (from the staged queue) will be queued in both the 382 * parallel encrypt and the serial encrypt queues; the incoming packets will 383 * be queued in both the parallel decrypt and the serial decrypt queues. 384 * 385 * - The parallel queues are used to distribute the encryption/decryption work 386 * across all CPUs. The per-CPU wg_{encrypt,decrypt}_worker() work on the 387 * parallel queues. 388 * - The serial queues ensure that packets are not reordered and are 389 * delivered in sequence for each peer. The per-peer wg_deliver_{in,out}() 390 * work on the serial queues. 391 */ 392 393 static void wg_queue_purge(struct wg_queue *); 394 395 static void 396 wg_queue_init(struct wg_queue *queue, const char *name) 397 { 398 lockinit(&queue->q_mtx, name, 0, 0); 399 STAILQ_INIT(&queue->q_queue); 400 queue->q_len = 0; 401 } 402 403 static void 404 wg_queue_deinit(struct wg_queue *queue) 405 { 406 wg_queue_purge(queue); 407 lockuninit(&queue->q_mtx); 408 } 409 410 static size_t 411 wg_queue_len(const struct wg_queue *queue) 412 { 413 return (queue->q_len); 414 } 415 416 static bool 417 wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt) 418 { 419 bool ok = false; 420 421 lockmgr(&hs->q_mtx, LK_EXCLUSIVE); 422 if (hs->q_len < MAX_QUEUED_HANDSHAKES) { 423 STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel); 424 hs->q_len++; 425 ok = true; 426 } 427 lockmgr(&hs->q_mtx, LK_RELEASE); 428 429 if (!ok) 430 wg_packet_free(pkt); 431 432 return (ok); 433 } 434 435 static struct wg_packet * 436 wg_queue_dequeue_handshake(struct wg_queue *hs) 437 { 438 struct wg_packet *pkt; 439 440 lockmgr(&hs->q_mtx, LK_EXCLUSIVE); 441 pkt = STAILQ_FIRST(&hs->q_queue); 442 if (pkt != NULL) { 443 STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel); 444 hs->q_len--; 445 } 446 lockmgr(&hs->q_mtx, LK_RELEASE); 447 448 return (pkt); 449 } 450 451 static void 452 wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt) 453 { 454 struct wg_packet *old = NULL; 455 456 lockmgr(&staged->q_mtx, LK_EXCLUSIVE); 457 if (staged->q_len >= MAX_STAGED_PKT) { 458 old = STAILQ_FIRST(&staged->q_queue); 459 STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel); 460 staged->q_len--; 461 } 462 STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel); 463 staged->q_len++; 464 lockmgr(&staged->q_mtx, LK_RELEASE); 465 466 if (old != NULL) 467 wg_packet_free(old); 468 } 469 470 static void 471 wg_queue_enlist_staged(struct wg_queue *staged, struct wg_packet_list *list) 472 { 473 struct wg_packet *pkt, *tpkt; 474 475 STAILQ_FOREACH_MUTABLE(pkt, list, p_parallel, tpkt) 476 wg_queue_push_staged(staged, pkt); 477 STAILQ_INIT(list); 478 } 479 480 static void 481 wg_queue_delist_staged(struct wg_queue *staged, struct wg_packet_list *list) 482 { 483 STAILQ_INIT(list); 484 lockmgr(&staged->q_mtx, LK_EXCLUSIVE); 485 STAILQ_CONCAT(list, &staged->q_queue); 486 staged->q_len = 0; 487 lockmgr(&staged->q_mtx, LK_RELEASE); 488 } 489 490 static void 491 wg_queue_purge(struct wg_queue *staged) 492 { 493 struct wg_packet_list list; 494 struct wg_packet *pkt, *tpkt; 495 496 wg_queue_delist_staged(staged, &list); 497 STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) 498 wg_packet_free(pkt); 499 } 500 501 static bool 502 wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, 503 struct wg_packet *pkt) 504 { 505 pkt->p_state = WG_PACKET_UNCRYPTED; 506 507 lockmgr(&serial->q_mtx, LK_EXCLUSIVE); 508 if (serial->q_len < MAX_QUEUED_PKT) { 509 serial->q_len++; 510 STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial); 511 } else { 512 lockmgr(&serial->q_mtx, LK_RELEASE); 513 wg_packet_free(pkt); 514 return (false); 515 } 516 lockmgr(&serial->q_mtx, LK_RELEASE); 517 518 lockmgr(¶llel->q_mtx, LK_EXCLUSIVE); 519 if (parallel->q_len < MAX_QUEUED_PKT) { 520 parallel->q_len++; 521 STAILQ_INSERT_TAIL(¶llel->q_queue, pkt, p_parallel); 522 } else { 523 lockmgr(¶llel->q_mtx, LK_RELEASE); 524 /* 525 * Cannot just free the packet because it's already queued 526 * in the serial queue. Instead, set its state to DEAD and 527 * let the serial worker to free it. 528 */ 529 pkt->p_state = WG_PACKET_DEAD; 530 return (false); 531 } 532 lockmgr(¶llel->q_mtx, LK_RELEASE); 533 534 return (true); 535 } 536 537 static struct wg_packet * 538 wg_queue_dequeue_serial(struct wg_queue *serial) 539 { 540 struct wg_packet *pkt = NULL; 541 542 lockmgr(&serial->q_mtx, LK_EXCLUSIVE); 543 if (serial->q_len > 0 && 544 STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) { 545 /* 546 * Dequeue both CRYPTED packets (to be delivered) and 547 * DEAD packets (to be freed). 548 */ 549 serial->q_len--; 550 pkt = STAILQ_FIRST(&serial->q_queue); 551 STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial); 552 } 553 lockmgr(&serial->q_mtx, LK_RELEASE); 554 555 return (pkt); 556 } 557 558 static struct wg_packet * 559 wg_queue_dequeue_parallel(struct wg_queue *parallel) 560 { 561 struct wg_packet *pkt = NULL; 562 563 lockmgr(¶llel->q_mtx, LK_EXCLUSIVE); 564 if (parallel->q_len > 0) { 565 parallel->q_len--; 566 pkt = STAILQ_FIRST(¶llel->q_queue); 567 STAILQ_REMOVE_HEAD(¶llel->q_queue, p_parallel); 568 } 569 lockmgr(¶llel->q_mtx, LK_RELEASE); 570 571 return (pkt); 572 } 573 574 /*----------------------------------------------------------------------------*/ 575 /* Peer */ 576 577 static struct wg_peer * 578 wg_peer_create(struct wg_softc *sc, const uint8_t pub_key[WG_KEY_SIZE]) 579 { 580 static unsigned long peer_counter = 0; 581 struct wg_peer *peer; 582 583 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 584 585 peer = kmalloc(sizeof(*peer), M_WG, M_WAITOK | M_ZERO); 586 587 peer->p_remote = noise_remote_alloc(sc->sc_local, pub_key, peer); 588 if (noise_remote_enable(peer->p_remote) != 0) { 589 kfree(peer, M_WG); 590 return (NULL); 591 } 592 593 peer->p_cookie = cookie_maker_alloc(pub_key); 594 595 peer->p_id = ++peer_counter; 596 peer->p_sc = sc; 597 peer->p_tx_bytes = kmalloc(sizeof(*peer->p_tx_bytes) * ncpus, 598 M_WG, M_WAITOK | M_ZERO); 599 peer->p_rx_bytes = kmalloc(sizeof(*peer->p_rx_bytes) * ncpus, 600 M_WG, M_WAITOK | M_ZERO); 601 602 lockinit(&peer->p_endpoint_lock, "wg_peer_endpoint", 0, 0); 603 lockinit(&peer->p_handshake_mtx, "wg_peer_handshake", 0, 0); 604 605 wg_queue_init(&peer->p_stage_queue, "stageq"); 606 wg_queue_init(&peer->p_encrypt_serial, "txq"); 607 wg_queue_init(&peer->p_decrypt_serial, "rxq"); 608 609 callout_init_mp(&peer->p_new_handshake); 610 callout_init_mp(&peer->p_send_keepalive); 611 callout_init_mp(&peer->p_retry_handshake); 612 callout_init_mp(&peer->p_persistent_keepalive); 613 callout_init_mp(&peer->p_zero_key_material); 614 615 TASK_INIT(&peer->p_send_task, 0, wg_deliver_out, peer); 616 TASK_INIT(&peer->p_recv_task, 0, wg_deliver_in, peer); 617 618 /* Randomly choose the taskqueues to distribute the load. */ 619 peer->p_send_taskqueue = wg_taskqueues[karc4random() % ncpus]; 620 peer->p_recv_taskqueue = wg_taskqueues[karc4random() % ncpus]; 621 622 LIST_INIT(&peer->p_aips); 623 624 TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry); 625 sc->sc_peers_num++; 626 627 if (sc->sc_ifp->if_link_state == LINK_STATE_UP) 628 wg_timers_enable(peer); 629 630 DPRINTF(sc, "Peer %ld created\n", peer->p_id); 631 return (peer); 632 } 633 634 static void 635 wg_peer_destroy(struct wg_peer *peer) 636 { 637 struct wg_softc *sc = peer->p_sc; 638 639 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 640 641 /* 642 * Disable remote and timers. This will prevent any new handshakes 643 * from occuring. 644 */ 645 noise_remote_disable(peer->p_remote); 646 wg_timers_disable(peer); 647 648 /* 649 * Remove all allowed IPs, so no more packets will be routed to 650 * this peer. 651 */ 652 wg_aip_remove_all(sc, peer); 653 654 /* Remove peer from the interface, then free. */ 655 sc->sc_peers_num--; 656 TAILQ_REMOVE(&sc->sc_peers, peer, p_entry); 657 658 /* 659 * While there are no references remaining, we may still have 660 * p_{send,recv}_task executing (think empty queue, but 661 * wg_deliver_{in,out} needs to check the queue). We should wait 662 * for them and then free. 663 */ 664 taskqueue_drain(peer->p_recv_taskqueue, &peer->p_recv_task); 665 taskqueue_drain(peer->p_send_taskqueue, &peer->p_send_task); 666 667 wg_queue_deinit(&peer->p_decrypt_serial); 668 wg_queue_deinit(&peer->p_encrypt_serial); 669 wg_queue_deinit(&peer->p_stage_queue); 670 671 kfree(peer->p_tx_bytes, M_WG); 672 kfree(peer->p_rx_bytes, M_WG); 673 674 lockuninit(&peer->p_endpoint_lock); 675 lockuninit(&peer->p_handshake_mtx); 676 677 noise_remote_free(peer->p_remote); 678 cookie_maker_free(peer->p_cookie); 679 680 DPRINTF(sc, "Peer %ld destroyed\n", peer->p_id); 681 kfree(peer, M_WG); 682 } 683 684 static void 685 wg_peer_destroy_all(struct wg_softc *sc) 686 { 687 struct wg_peer *peer, *tpeer; 688 689 TAILQ_FOREACH_MUTABLE(peer, &sc->sc_peers, p_entry, tpeer) 690 wg_peer_destroy(peer); 691 } 692 693 static int 694 wg_peer_set_sockaddr(struct wg_peer *peer, const struct sockaddr *remote) 695 { 696 int ret = 0; 697 698 lockmgr(&peer->p_endpoint_lock, LK_EXCLUSIVE); 699 700 memcpy(&peer->p_endpoint.e_remote, remote, 701 sizeof(peer->p_endpoint.e_remote)); 702 if (remote->sa_family == AF_INET) 703 memcpy(&peer->p_endpoint.e_remote.r_sin, remote, 704 sizeof(peer->p_endpoint.e_remote.r_sin)); 705 #ifdef INET6 706 else if (remote->sa_family == AF_INET6) 707 memcpy(&peer->p_endpoint.e_remote.r_sin6, remote, 708 sizeof(peer->p_endpoint.e_remote.r_sin6)); 709 #endif 710 else 711 ret = EAFNOSUPPORT; 712 713 /* No 'e_local' to clear on DragonFly. */ 714 715 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 716 return (ret); 717 } 718 719 static int 720 wg_peer_get_sockaddr(struct wg_peer *peer, struct sockaddr *remote) 721 { 722 int ret = ENOENT; 723 724 lockmgr(&peer->p_endpoint_lock, LK_SHARED); 725 if (peer->p_endpoint.e_remote.r_sa.sa_family != AF_UNSPEC) { 726 memcpy(remote, &peer->p_endpoint.e_remote, 727 sizeof(peer->p_endpoint.e_remote)); 728 ret = 0; 729 } 730 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 731 return (ret); 732 } 733 734 static void 735 wg_peer_set_endpoint(struct wg_peer *peer, const struct wg_endpoint *e) 736 { 737 KKASSERT(e->e_remote.r_sa.sa_family != AF_UNSPEC); 738 739 if (__predict_true(memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)) 740 return; 741 742 lockmgr(&peer->p_endpoint_lock, LK_EXCLUSIVE); 743 peer->p_endpoint = *e; 744 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 745 } 746 747 static void 748 wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *e) 749 { 750 if (__predict_true(memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)) 751 return; 752 753 lockmgr(&peer->p_endpoint_lock, LK_SHARED); 754 *e = peer->p_endpoint; 755 lockmgr(&peer->p_endpoint_lock, LK_RELEASE); 756 } 757 758 /*----------------------------------------------------------------------------*/ 759 /* Allowed IP */ 760 761 static int 762 wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, 763 const void *addr, uint8_t cidr) 764 { 765 struct radix_node_head *head; 766 struct radix_node *node; 767 struct wg_aip *aip; 768 int ret = 0; 769 770 aip = kmalloc(sizeof(*aip), M_WG, M_WAITOK | M_ZERO); 771 aip->a_peer = peer; 772 aip->a_af = af; 773 774 switch (af) { 775 case AF_INET: 776 if (cidr > 32) 777 cidr = 32; 778 head = sc->sc_aip4; 779 aip->a_addr.in = *(const struct in_addr *)addr; 780 aip->a_mask.ip = 781 htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff); 782 aip->a_addr.ip &= aip->a_mask.ip; 783 aip->a_addr.length = aip->a_mask.length = 784 offsetof(struct aip_addr, in) + sizeof(struct in_addr); 785 break; 786 #ifdef INET6 787 case AF_INET6: 788 { 789 int i; 790 791 if (cidr > 128) 792 cidr = 128; 793 head = sc->sc_aip6; 794 aip->a_addr.in6 = *(const struct in6_addr *)addr; 795 in6_prefixlen2mask(&aip->a_mask.in6, cidr); 796 for (i = 0; i < 4; i++) 797 aip->a_addr.ip6[i] &= aip->a_mask.ip6[i]; 798 aip->a_addr.length = aip->a_mask.length = 799 offsetof(struct aip_addr, in6) + sizeof(struct in6_addr); 800 break; 801 } 802 #endif 803 default: 804 kfree(aip, M_WG); 805 return (EAFNOSUPPORT); 806 } 807 808 lockmgr(&sc->sc_aip_lock, LK_EXCLUSIVE); 809 node = head->rnh_addaddr(&aip->a_addr, &aip->a_mask, head, 810 aip->a_nodes); 811 if (node != NULL) { 812 KKASSERT(node == aip->a_nodes); 813 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); 814 peer->p_aips_num++; 815 } else { 816 /* 817 * Two possibilities: 818 * - out of memory failure 819 * - entry already exists 820 */ 821 node = head->rnh_lookup(&aip->a_addr, &aip->a_mask, head); 822 if (node == NULL) { 823 kfree(aip, M_WG); 824 ret = ENOMEM; 825 } else { 826 KKASSERT(node != aip->a_nodes); 827 kfree(aip, M_WG); 828 aip = (struct wg_aip *)node; 829 if (aip->a_peer != peer) { 830 /* Replace the peer. */ 831 LIST_REMOVE(aip, a_entry); 832 aip->a_peer->p_aips_num--; 833 aip->a_peer = peer; 834 LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry); 835 aip->a_peer->p_aips_num++; 836 } 837 } 838 } 839 lockmgr(&sc->sc_aip_lock, LK_RELEASE); 840 841 return (ret); 842 } 843 844 static struct wg_peer * 845 wg_aip_lookup(struct wg_softc *sc, sa_family_t af, const void *a) 846 { 847 struct radix_node_head *head; 848 struct radix_node *node; 849 struct wg_peer *peer; 850 struct aip_addr addr; 851 852 switch (af) { 853 case AF_INET: 854 head = sc->sc_aip4; 855 memcpy(&addr.in, a, sizeof(addr.in)); 856 addr.length = offsetof(struct aip_addr, in) + sizeof(addr.in); 857 break; 858 case AF_INET6: 859 head = sc->sc_aip6; 860 memcpy(&addr.in6, a, sizeof(addr.in6)); 861 addr.length = offsetof(struct aip_addr, in6) + sizeof(addr.in6); 862 break; 863 default: 864 return (NULL); 865 } 866 867 lockmgr(&sc->sc_aip_lock, LK_SHARED); 868 node = head->rnh_matchaddr(&addr, head); 869 if (node != NULL) { 870 peer = ((struct wg_aip *)node)->a_peer; 871 noise_remote_ref(peer->p_remote); 872 } else { 873 peer = NULL; 874 } 875 lockmgr(&sc->sc_aip_lock, LK_RELEASE); 876 877 return (peer); 878 } 879 880 static void 881 wg_aip_remove_all(struct wg_softc *sc, struct wg_peer *peer) 882 { 883 struct radix_node_head *head; 884 struct radix_node *node; 885 struct wg_aip *aip, *taip; 886 887 lockmgr(&sc->sc_aip_lock, LK_EXCLUSIVE); 888 889 LIST_FOREACH_MUTABLE(aip, &peer->p_aips, a_entry, taip) { 890 switch (aip->a_af) { 891 case AF_INET: 892 head = sc->sc_aip4; 893 break; 894 case AF_INET6: 895 head = sc->sc_aip6; 896 break; 897 default: 898 panic("%s: impossible aip %p", __func__, aip); 899 } 900 node = head->rnh_deladdr(&aip->a_addr, &aip->a_mask, head); 901 if (node == NULL) 902 panic("%s: failed to delete aip %p", __func__, aip); 903 LIST_REMOVE(aip, a_entry); 904 peer->p_aips_num--; 905 kfree(aip, M_WG); 906 } 907 908 if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0) 909 panic("%s: could not delete all aips for peer %ld", 910 __func__, peer->p_id); 911 912 lockmgr(&sc->sc_aip_lock, LK_RELEASE); 913 } 914 915 /*----------------------------------------------------------------------------*/ 916 /* Socket */ 917 918 static int wg_socket_open(struct socket **, sa_family_t, in_port_t *, 919 void *); 920 static int wg_socket_set_sockopt(struct socket *, struct socket *, 921 int, void *, size_t); 922 923 static int 924 wg_socket_init(struct wg_softc *sc, in_port_t port) 925 { 926 struct wg_socket *so = &sc->sc_socket; 927 struct socket *so4 = NULL, *so6 = NULL; 928 in_port_t bound_port = port; 929 uint32_t cookie; 930 int ret; 931 932 /* 933 * When a host or a jail doesn't support the AF, sobind() would 934 * return EADDRNOTAVAIL. Handle this case in order to support such 935 * IPv4-only or IPv6-only environments. 936 * 937 * However, in a dual-stack environment, both IPv4 and IPv6 sockets 938 * must bind the same port. 939 */ 940 ret = wg_socket_open(&so4, AF_INET, &bound_port, sc); 941 if (ret != 0 && ret != EADDRNOTAVAIL) 942 goto error; 943 944 #ifdef INET6 945 ret = wg_socket_open(&so6, AF_INET6, &bound_port, sc); 946 if (ret != 0 && ret != EADDRNOTAVAIL) 947 goto error; 948 #endif 949 950 if (so4 == NULL && so6 == NULL) { 951 ret = EAFNOSUPPORT; 952 goto error; 953 } 954 955 cookie = so->so_user_cookie; 956 if (cookie != 0) { 957 ret = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, 958 &cookie, sizeof(cookie)); 959 if (ret != 0) 960 goto error; 961 } 962 963 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 964 965 lockinit(&so->so_lock, "wg socket lock", 0, 0); 966 967 if (so->so_so4 != NULL) 968 soclose(so->so_so4, 0); 969 if (so->so_so6 != NULL) 970 soclose(so->so_so6, 0); 971 so->so_so4 = so4; 972 so->so_so6 = so6; 973 so->so_port = bound_port; 974 975 return (0); 976 977 error: 978 if (so4 != NULL) 979 soclose(so4, 0); 980 if (so6 != NULL) 981 soclose(so6, 0); 982 return (ret); 983 } 984 985 static int 986 wg_socket_open(struct socket **so, sa_family_t af, in_port_t *port, 987 void *upcall_arg) 988 { 989 struct sockaddr_in sin; 990 #ifdef INET6 991 struct sockaddr_in6 sin6; 992 #endif 993 struct sockaddr *sa, *bound_sa; 994 int ret; 995 996 if (af == AF_INET) { 997 bzero(&sin, sizeof(sin)); 998 sin.sin_len = sizeof(struct sockaddr_in); 999 sin.sin_family = AF_INET; 1000 sin.sin_port = htons(*port); 1001 sa = sintosa(&sin); 1002 #ifdef INET6 1003 } else if (af == AF_INET6) { 1004 bzero(&sin6, sizeof(sin6)); 1005 sin6.sin6_len = sizeof(struct sockaddr_in6); 1006 sin6.sin6_family = AF_INET6; 1007 sin6.sin6_port = htons(*port); 1008 sa = sintosa(&sin6); 1009 #endif 1010 } else { 1011 return (EAFNOSUPPORT); 1012 } 1013 1014 ret = socreate(af, so, SOCK_DGRAM, IPPROTO_UDP, curthread); 1015 if (ret != 0) 1016 return (ret); 1017 1018 (*so)->so_upcall = wg_upcall; 1019 (*so)->so_upcallarg = upcall_arg; 1020 atomic_set_int(&(*so)->so_rcv.ssb_flags, SSB_UPCALL); 1021 1022 ret = sobind(*so, sa, curthread); 1023 if (ret != 0) 1024 goto error; 1025 1026 if (*port == 0) { 1027 ret = so_pru_sockaddr(*so, &bound_sa); 1028 if (ret != 0) 1029 goto error; 1030 if (bound_sa->sa_family == AF_INET) 1031 *port = ntohs(satosin(bound_sa)->sin_port); 1032 else 1033 *port = ntohs(satosin6(bound_sa)->sin6_port); 1034 kfree(bound_sa, M_SONAME); 1035 } 1036 1037 return (0); 1038 1039 error: 1040 if (*so != NULL) { 1041 soclose(*so, 0); 1042 *so = NULL; 1043 } 1044 return (ret); 1045 } 1046 1047 static void 1048 wg_socket_uninit(struct wg_softc *sc) 1049 { 1050 struct wg_socket *so = &sc->sc_socket; 1051 1052 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 1053 1054 lockmgr(&so->so_lock, LK_EXCLUSIVE); 1055 1056 if (so->so_so4 != NULL) { 1057 soclose(so->so_so4, 0); 1058 so->so_so4 = NULL; 1059 } 1060 if (so->so_so6 != NULL) { 1061 soclose(so->so_so6, 0); 1062 so->so_so6 = NULL; 1063 } 1064 1065 lockmgr(&so->so_lock, LK_RELEASE); 1066 lockuninit(&so->so_lock); 1067 } 1068 1069 static int 1070 wg_socket_set_sockopt(struct socket *so4, struct socket *so6, 1071 int name, void *val, size_t len) 1072 { 1073 struct sockopt sopt = { 1074 .sopt_dir = SOPT_SET, 1075 .sopt_level = SOL_SOCKET, 1076 .sopt_name = name, 1077 .sopt_val = val, 1078 .sopt_valsize = len, 1079 }; 1080 int ret; 1081 1082 if (so4 != NULL) { 1083 ret = sosetopt(so4, &sopt); 1084 if (ret != 0) 1085 return (ret); 1086 } 1087 if (so6 != NULL) { 1088 ret = sosetopt(so6, &sopt); 1089 if (ret != 0) 1090 return (ret); 1091 } 1092 1093 return (0); 1094 } 1095 1096 static int 1097 wg_socket_set_cookie(struct wg_softc *sc, uint32_t user_cookie) 1098 { 1099 struct wg_socket *so; 1100 int ret; 1101 1102 KKASSERT(lockstatus(&sc->sc_lock, curthread) == LK_EXCLUSIVE); 1103 1104 so = &sc->sc_socket; 1105 lockmgr(&so->so_lock, LK_EXCLUSIVE); 1106 1107 ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, 1108 &user_cookie, sizeof(user_cookie)); 1109 if (ret == 0) 1110 so->so_user_cookie = user_cookie; 1111 1112 lockmgr(&so->so_lock, LK_RELEASE); 1113 return (ret); 1114 } 1115 1116 static int 1117 wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m) 1118 { 1119 struct wg_socket *so; 1120 struct sockaddr *sa; 1121 int len, ret; 1122 1123 so = &sc->sc_socket; 1124 sa = &e->e_remote.r_sa; 1125 len = m->m_pkthdr.len; 1126 ret = 0; 1127 1128 /* 1129 * NOTE: DragonFly by default sends UDP packets asynchronously, 1130 * unless the 'net.inet.udp.sosend_async' sysctl MIB is set 1131 * to 0 or the 'MSG_SYNC' flag is set for so_pru_sosend(). 1132 * And in the async mode, an error code cannot really be 1133 * replied to the caller. So so_pru_sosend() may return 0 1134 * even if the packet fails to send. 1135 */ 1136 lockmgr(&so->so_lock, LK_SHARED); 1137 if (sa->sa_family == AF_INET && so->so_so4 != NULL) { 1138 ret = so_pru_sosend(so->so_so4, sa, NULL /* uio */, 1139 m, NULL /* control */, 0 /* flags */, 1140 curthread); 1141 #ifdef INET6 1142 } else if (sa->sa_family == AF_INET6 && so->so_so6 != NULL) { 1143 ret = so_pru_sosend(so->so_so6, sa, NULL /* uio */, 1144 m, NULL /* control */, 0 /* flags */, 1145 curthread); 1146 #endif 1147 } else { 1148 ret = ENOTCONN; 1149 m_freem(m); 1150 } 1151 lockmgr(&so->so_lock, LK_RELEASE); 1152 1153 if (ret == 0) { 1154 IFNET_STAT_INC(sc->sc_ifp, opackets, 1); 1155 IFNET_STAT_INC(sc->sc_ifp, obytes, len); 1156 } else { 1157 IFNET_STAT_INC(sc->sc_ifp, oerrors, 1); 1158 } 1159 1160 return (ret); 1161 } 1162 1163 static void 1164 wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, const void *buf, 1165 size_t len) 1166 { 1167 struct mbuf *m; 1168 int ret; 1169 1170 /* 1171 * This function only sends handshake packets of known lengths that 1172 * are <= MHLEN, so it's safe to just use m_gethdr() and memcpy(). 1173 */ 1174 KKASSERT(len <= MHLEN); 1175 1176 m = m_gethdr(M_NOWAIT, MT_DATA); 1177 if (m == NULL) { 1178 DPRINTF(sc, "Unable to allocate mbuf\n"); 1179 return; 1180 } 1181 1182 /* Just plain copy as it's a single mbuf. */ 1183 memcpy(mtod(m, void *), buf, len); 1184 m->m_pkthdr.len = m->m_len = len; 1185 1186 /* Give high priority to the handshake packets. */ 1187 m->m_flags |= M_PRIO; 1188 1189 ret = wg_send(sc, e, m); 1190 if (ret != 0) 1191 DPRINTF(sc, "Unable to send packet: %d\n", ret); 1192 } 1193 1194 /*----------------------------------------------------------------------------*/ 1195 /* 1196 * Timers 1197 * 1198 * These functions handle the timeout callbacks for a WireGuard session, and 1199 * provide an "event-based" model for controlling WireGuard session timers. 1200 */ 1201 1202 static void wg_timers_run_send_initiation(struct wg_peer *, bool); 1203 static void wg_timers_run_retry_handshake(void *); 1204 static void wg_timers_run_send_keepalive(void *); 1205 static void wg_timers_run_new_handshake(void *); 1206 static void wg_timers_run_zero_key_material(void *); 1207 static void wg_timers_run_persistent_keepalive(void *); 1208 1209 static void 1210 wg_timers_enable(struct wg_peer *peer) 1211 { 1212 atomic_store_bool(&peer->p_enabled, true); 1213 wg_timers_run_persistent_keepalive(peer); 1214 } 1215 1216 static void 1217 wg_timers_disable(struct wg_peer *peer) 1218 { 1219 atomic_store_bool(&peer->p_enabled, false); 1220 atomic_store_bool(&peer->p_need_another_keepalive, false); 1221 1222 /* Cancel the callouts and wait for them to complete. */ 1223 callout_drain(&peer->p_new_handshake); 1224 callout_drain(&peer->p_send_keepalive); 1225 callout_drain(&peer->p_retry_handshake); 1226 callout_drain(&peer->p_persistent_keepalive); 1227 callout_drain(&peer->p_zero_key_material); 1228 } 1229 1230 static void 1231 wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval) 1232 { 1233 atomic_store_16(&peer->p_persistent_keepalive_interval, interval); 1234 if (atomic_load_bool(&peer->p_enabled)) 1235 wg_timers_run_persistent_keepalive(peer); 1236 } 1237 1238 static bool 1239 wg_timers_get_persistent_keepalive(struct wg_peer *peer, uint16_t *interval) 1240 { 1241 *interval = atomic_load_16(&peer->p_persistent_keepalive_interval); 1242 return (*interval > 0); 1243 } 1244 1245 static void 1246 wg_timers_get_last_handshake(struct wg_peer *peer, struct timespec *time) 1247 { 1248 lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); 1249 *time = peer->p_handshake_complete; 1250 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1251 } 1252 1253 /* 1254 * Should be called after an authenticated data packet is sent. 1255 */ 1256 static void 1257 wg_timers_event_data_sent(struct wg_peer *peer) 1258 { 1259 int ticks; 1260 1261 if (atomic_load_bool(&peer->p_enabled) && 1262 !callout_pending(&peer->p_new_handshake)) { 1263 ticks = NEW_HANDSHAKE_TIMEOUT * hz + 1264 REKEY_TIMEOUT_JITTER * hz / 1000; 1265 callout_reset(&peer->p_new_handshake, ticks, 1266 wg_timers_run_new_handshake, peer); 1267 } 1268 } 1269 1270 /* 1271 * Should be called after an authenticated data packet is received. 1272 */ 1273 static void 1274 wg_timers_event_data_received(struct wg_peer *peer) 1275 { 1276 if (atomic_load_bool(&peer->p_enabled)) { 1277 if (!callout_pending(&peer->p_send_keepalive)) { 1278 callout_reset(&peer->p_send_keepalive, 1279 KEEPALIVE_TIMEOUT * hz, 1280 wg_timers_run_send_keepalive, peer); 1281 } else { 1282 atomic_store_bool(&peer->p_need_another_keepalive, 1283 true); 1284 } 1285 } 1286 } 1287 1288 /* 1289 * Should be called before any type of authenticated packet is to be sent, 1290 * whether keepalive, data, or handshake. 1291 */ 1292 static void 1293 wg_timers_event_any_authenticated_packet_sent(struct wg_peer *peer) 1294 { 1295 callout_stop(&peer->p_send_keepalive); 1296 } 1297 1298 /* 1299 * Should be called after any type of authenticated packet is received, 1300 * whether keepalive, data, or handshake. 1301 */ 1302 static void 1303 wg_timers_event_any_authenticated_packet_received(struct wg_peer *peer) 1304 { 1305 callout_stop(&peer->p_new_handshake); 1306 } 1307 1308 /* 1309 * Should be called before a packet with authentication (whether keepalive, 1310 * data, or handshakem) is sent, or after one is received. 1311 */ 1312 static void 1313 wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer) 1314 { 1315 uint16_t interval; 1316 1317 interval = atomic_load_16(&peer->p_persistent_keepalive_interval); 1318 if (atomic_load_bool(&peer->p_enabled) && interval > 0) { 1319 callout_reset(&peer->p_persistent_keepalive, interval * hz, 1320 wg_timers_run_persistent_keepalive, peer); 1321 } 1322 } 1323 1324 /* 1325 * Should be called after a handshake initiation message is sent. 1326 */ 1327 static void 1328 wg_timers_event_handshake_initiated(struct wg_peer *peer) 1329 { 1330 int ticks; 1331 1332 if (atomic_load_bool(&peer->p_enabled)) { 1333 ticks = REKEY_TIMEOUT * hz + REKEY_TIMEOUT_JITTER * hz / 1000; 1334 callout_reset(&peer->p_retry_handshake, ticks, 1335 wg_timers_run_retry_handshake, peer); 1336 } 1337 } 1338 1339 /* 1340 * Should be called after a handshake response message is received and 1341 * processed, or when getting key confirmation via the first data message. 1342 */ 1343 static void 1344 wg_timers_event_handshake_complete(struct wg_peer *peer) 1345 { 1346 if (atomic_load_bool(&peer->p_enabled)) { 1347 lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); 1348 callout_stop(&peer->p_retry_handshake); 1349 peer->p_handshake_retries = 0; 1350 getnanotime(&peer->p_handshake_complete); 1351 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1352 1353 wg_timers_run_send_keepalive(peer); 1354 } 1355 } 1356 1357 /* 1358 * Should be called after an ephemeral key is created, which is before sending 1359 * a handshake response or after receiving a handshake response. 1360 */ 1361 static void 1362 wg_timers_event_session_derived(struct wg_peer *peer) 1363 { 1364 if (atomic_load_bool(&peer->p_enabled)) { 1365 callout_reset(&peer->p_zero_key_material, 1366 REJECT_AFTER_TIME * 3 * hz, 1367 wg_timers_run_zero_key_material, peer); 1368 } 1369 } 1370 1371 /* 1372 * Should be called after data packet sending failure, or after the old 1373 * keypairs expiring (or near expiring). 1374 */ 1375 static void 1376 wg_timers_event_want_initiation(struct wg_peer *peer) 1377 { 1378 if (atomic_load_bool(&peer->p_enabled)) 1379 wg_timers_run_send_initiation(peer, false); 1380 } 1381 1382 static void 1383 wg_timers_run_send_initiation(struct wg_peer *peer, bool is_retry) 1384 { 1385 if (!is_retry) 1386 peer->p_handshake_retries = 0; 1387 if (noise_remote_initiation_expired(peer->p_remote)) 1388 wg_send_initiation(peer); 1389 } 1390 1391 static void 1392 wg_timers_run_retry_handshake(void *_peer) 1393 { 1394 struct wg_peer *peer = _peer; 1395 1396 lockmgr(&peer->p_handshake_mtx, LK_EXCLUSIVE); 1397 if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) { 1398 peer->p_handshake_retries++; 1399 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1400 1401 DPRINTF(peer->p_sc, "Handshake for peer %ld did not complete " 1402 "after %d seconds, retrying (try %d)\n", peer->p_id, 1403 REKEY_TIMEOUT, peer->p_handshake_retries + 1); 1404 wg_timers_run_send_initiation(peer, true); 1405 } else { 1406 lockmgr(&peer->p_handshake_mtx, LK_RELEASE); 1407 1408 DPRINTF(peer->p_sc, "Handshake for peer %ld did not complete " 1409 "after %d retries, giving up\n", peer->p_id, 1410 MAX_TIMER_HANDSHAKES + 2); 1411 callout_stop(&peer->p_send_keepalive); 1412 wg_queue_purge(&peer->p_stage_queue); 1413 if (atomic_load_bool(&peer->p_enabled) && 1414 !callout_pending(&peer->p_zero_key_material)) { 1415 callout_reset(&peer->p_zero_key_material, 1416 REJECT_AFTER_TIME * 3 * hz, 1417 wg_timers_run_zero_key_material, peer); 1418 } 1419 } 1420 } 1421 1422 static void 1423 wg_timers_run_send_keepalive(void *_peer) 1424 { 1425 struct wg_peer *peer = _peer; 1426 1427 wg_send_keepalive(peer); 1428 1429 if (atomic_load_bool(&peer->p_enabled) && 1430 atomic_load_bool(&peer->p_need_another_keepalive)) { 1431 atomic_store_bool(&peer->p_need_another_keepalive, false); 1432 callout_reset(&peer->p_send_keepalive, KEEPALIVE_TIMEOUT * hz, 1433 wg_timers_run_send_keepalive, peer); 1434 } 1435 } 1436 1437 static void 1438 wg_timers_run_persistent_keepalive(void *_peer) 1439 { 1440 struct wg_peer *peer = _peer; 1441 1442 if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0) 1443 wg_send_keepalive(peer); 1444 } 1445 1446 static void 1447 wg_timers_run_new_handshake(void *_peer) 1448 { 1449 struct wg_peer *peer = _peer; 1450 1451 DPRINTF(peer->p_sc, "Retrying handshake with peer %ld, " 1452 "because we stopped hearing back after %d seconds\n", 1453 peer->p_id, NEW_HANDSHAKE_TIMEOUT); 1454 wg_timers_run_send_initiation(peer, false); 1455 } 1456 1457 static void 1458 wg_timers_run_zero_key_material(void *_peer) 1459 { 1460 struct wg_peer *peer = _peer; 1461 1462 DPRINTF(peer->p_sc, "Zeroing out keys for peer %ld, " 1463 "since we haven't received a new one in %d seconds\n", 1464 peer->p_id, REJECT_AFTER_TIME * 3); 1465 noise_remote_keypairs_clear(peer->p_remote); 1466 } 1467 1468 /*----------------------------------------------------------------------------*/ 1469 /* Handshake */ 1470 1471 static void 1472 wg_peer_send_buf(struct wg_peer *peer, const void *buf, size_t len) 1473 { 1474 struct wg_endpoint endpoint; 1475 1476 peer->p_tx_bytes[mycpuid] += len; 1477 1478 wg_timers_event_any_authenticated_packet_traversal(peer); 1479 wg_timers_event_any_authenticated_packet_sent(peer); 1480 1481 wg_peer_get_endpoint(peer, &endpoint); 1482 wg_send_buf(peer->p_sc, &endpoint, buf, len); 1483 } 1484 1485 static void 1486 wg_send_initiation(struct wg_peer *peer) 1487 { 1488 struct wg_pkt_initiation pkt; 1489 1490 if (!noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue, 1491 pkt.es, pkt.ets)) 1492 return; 1493 1494 DPRINTF(peer->p_sc, "Sending handshake initiation to peer %ld\n", 1495 peer->p_id); 1496 1497 pkt.t = WG_PKT_INITIATION; 1498 cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt, 1499 sizeof(pkt) - sizeof(pkt.m)); 1500 wg_peer_send_buf(peer, &pkt, sizeof(pkt)); 1501 wg_timers_event_handshake_initiated(peer); 1502 } 1503 1504 static void 1505 wg_send_response(struct wg_peer *peer) 1506 { 1507 struct wg_pkt_response pkt; 1508 1509 if (!noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx, 1510 pkt.ue, pkt.en)) 1511 return; 1512 1513 DPRINTF(peer->p_sc, "Sending handshake response to peer %ld\n", 1514 peer->p_id); 1515 1516 wg_timers_event_session_derived(peer); 1517 pkt.t = WG_PKT_RESPONSE; 1518 cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt, 1519 sizeof(pkt) - sizeof(pkt.m)); 1520 wg_peer_send_buf(peer, &pkt, sizeof(pkt)); 1521 } 1522 1523 static void 1524 wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx, 1525 struct wg_endpoint *e) 1526 { 1527 struct wg_pkt_cookie pkt; 1528 1529 DPRINTF(sc, "Sending cookie response for denied handshake message\n"); 1530 1531 pkt.t = WG_PKT_COOKIE; 1532 pkt.r_idx = idx; 1533 1534 cookie_checker_create_payload(sc->sc_cookie, cm, pkt.nonce, 1535 pkt.ec, &e->e_remote.r_sa); 1536 wg_send_buf(sc, e, &pkt, sizeof(pkt)); 1537 } 1538 1539 static void 1540 wg_send_keepalive(struct wg_peer *peer) 1541 { 1542 struct wg_packet *pkt; 1543 struct mbuf *m; 1544 1545 if (wg_queue_len(&peer->p_stage_queue) > 0) 1546 goto send; 1547 if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 1548 return; 1549 if ((pkt = wg_packet_alloc(m)) == NULL) { 1550 m_freem(m); 1551 return; 1552 } 1553 1554 wg_queue_push_staged(&peer->p_stage_queue, pkt); 1555 DPRINTF(peer->p_sc, "Sending keepalive packet to peer %ld\n", 1556 peer->p_id); 1557 send: 1558 wg_peer_send_staged(peer); 1559 } 1560 1561 static bool 1562 wg_is_underload(struct wg_softc *sc) 1563 { 1564 /* 1565 * This is global, so that the load calculation applies to the 1566 * whole system. Don't care about races with it at all. 1567 */ 1568 static struct timespec last_underload; /* nanouptime */ 1569 struct timespec now; 1570 bool underload; 1571 1572 underload = (wg_queue_len(&sc->sc_handshake_queue) >= 1573 MAX_QUEUED_HANDSHAKES / 8); 1574 if (underload) { 1575 getnanouptime(&last_underload); 1576 } else if (timespecisset(&last_underload)) { 1577 getnanouptime(&now); 1578 now.tv_sec -= UNDERLOAD_TIMEOUT; 1579 underload = timespeccmp(&last_underload, &now, >); 1580 if (!underload) 1581 timespecclear(&last_underload); 1582 } 1583 1584 return (underload); 1585 } 1586 1587 static void 1588 wg_handshake(struct wg_softc *sc, struct wg_packet *pkt) 1589 { 1590 struct wg_pkt_initiation *init; 1591 struct wg_pkt_response *resp; 1592 struct wg_pkt_cookie *cook; 1593 struct wg_endpoint *e; 1594 struct wg_peer *peer; 1595 struct mbuf *m; 1596 struct noise_remote *remote = NULL; 1597 bool underload; 1598 int ret; 1599 1600 underload = wg_is_underload(sc); 1601 m = pkt->p_mbuf; 1602 e = &pkt->p_endpoint; 1603 1604 if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL) 1605 goto error; 1606 1607 switch (*mtod(m, uint32_t *)) { 1608 case WG_PKT_INITIATION: 1609 init = mtod(m, struct wg_pkt_initiation *); 1610 1611 ret = cookie_checker_validate_macs(sc->sc_cookie, &init->m, 1612 init, sizeof(*init) - sizeof(init->m), underload, 1613 &e->e_remote.r_sa); 1614 if (ret != 0) { 1615 switch (ret) { 1616 case EINVAL: 1617 DPRINTF(sc, "Invalid initiation MAC\n"); 1618 break; 1619 case ECONNREFUSED: 1620 DPRINTF(sc, "Handshake ratelimited\n"); 1621 break; 1622 case EAGAIN: 1623 wg_send_cookie(sc, &init->m, init->s_idx, e); 1624 break; 1625 default: 1626 /* 1627 * cookie_checker_validate_macs() seems could 1628 * return EAFNOSUPPORT, but that is actually 1629 * impossible, because packets of unsupported 1630 * AF have been already dropped. 1631 */ 1632 panic("%s: unexpected return: %d", 1633 __func__, ret); 1634 } 1635 goto error; 1636 } 1637 1638 remote = noise_consume_initiation(sc->sc_local, init->s_idx, 1639 init->ue, init->es, 1640 init->ets); 1641 if (remote == NULL) { 1642 DPRINTF(sc, "Invalid handshake initiation\n"); 1643 goto error; 1644 } 1645 1646 peer = noise_remote_arg(remote); 1647 DPRINTF(sc, "Receiving handshake initiation from peer %ld\n", 1648 peer->p_id); 1649 1650 wg_peer_set_endpoint(peer, e); 1651 wg_send_response(peer); 1652 break; 1653 1654 case WG_PKT_RESPONSE: 1655 resp = mtod(m, struct wg_pkt_response *); 1656 1657 ret = cookie_checker_validate_macs(sc->sc_cookie, &resp->m, 1658 resp, sizeof(*resp) - sizeof(resp->m), underload, 1659 &e->e_remote.r_sa); 1660 if (ret != 0) { 1661 switch (ret) { 1662 case EINVAL: 1663 DPRINTF(sc, "Invalid response MAC\n"); 1664 break; 1665 case ECONNREFUSED: 1666 DPRINTF(sc, "Handshake ratelimited\n"); 1667 break; 1668 case EAGAIN: 1669 wg_send_cookie(sc, &resp->m, resp->s_idx, e); 1670 break; 1671 default: 1672 /* See also the comment above. */ 1673 panic("%s: unexpected return: %d", 1674 __func__, ret); 1675 } 1676 goto error; 1677 } 1678 1679 remote = noise_consume_response(sc->sc_local, resp->s_idx, 1680 resp->r_idx, resp->ue, 1681 resp->en); 1682 if (remote == NULL) { 1683 DPRINTF(sc, "Invalid handshake response\n"); 1684 goto error; 1685 } 1686 1687 peer = noise_remote_arg(remote); 1688 DPRINTF(sc, "Receiving handshake response from peer %ld\n", 1689 peer->p_id); 1690 1691 wg_peer_set_endpoint(peer, e); 1692 wg_timers_event_session_derived(peer); 1693 wg_timers_event_handshake_complete(peer); 1694 break; 1695 1696 case WG_PKT_COOKIE: 1697 cook = mtod(m, struct wg_pkt_cookie *); 1698 1699 remote = noise_remote_index(sc->sc_local, cook->r_idx); 1700 if (remote == NULL) { 1701 DPRINTF(sc, "Unknown cookie index\n"); 1702 goto error; 1703 } 1704 1705 peer = noise_remote_arg(remote); 1706 if (cookie_maker_consume_payload(peer->p_cookie, cook->nonce, 1707 cook->ec) == 0) { 1708 DPRINTF(sc, "Receiving cookie response\n"); 1709 } else { 1710 DPRINTF(sc, "Could not decrypt cookie response\n"); 1711 goto error; 1712 } 1713 1714 goto not_authenticated; 1715 1716 default: 1717 panic("%s: invalid packet in handshake queue", __func__); 1718 } 1719 1720 wg_timers_event_any_authenticated_packet_received(peer); 1721 wg_timers_event_any_authenticated_packet_traversal(peer); 1722 1723 not_authenticated: 1724 peer->p_rx_bytes[mycpuid] += m->m_pkthdr.len; 1725 IFNET_STAT_INC(sc->sc_ifp, ipackets, 1); 1726 IFNET_STAT_INC(sc->sc_ifp, ibytes, m->m_pkthdr.len); 1727 error: 1728 if (remote != NULL) 1729 noise_remote_put(remote); 1730 wg_packet_free(pkt); 1731 } 1732 1733 static void 1734 wg_handshake_worker(void *arg, int pending __unused) 1735 { 1736 struct wg_softc *sc = arg; 1737 struct wg_queue *queue = &sc->sc_handshake_queue; 1738 struct wg_packet *pkt; 1739 1740 while ((pkt = wg_queue_dequeue_handshake(queue)) != NULL) 1741 wg_handshake(sc, pkt); 1742 } 1743 1744 /*----------------------------------------------------------------------------*/ 1745 /* Transport Packet Functions */ 1746 1747 static inline void 1748 wg_bpf_ptap(struct ifnet *ifp, struct mbuf *m, sa_family_t af) 1749 { 1750 uint32_t bpf_af; 1751 1752 if (ifp->if_bpf == NULL) 1753 return; 1754 1755 bpf_gettoken(); 1756 /* Double check after obtaining the token. */ 1757 if (ifp->if_bpf != NULL) { 1758 /* Prepend the AF as a 4-byte field for DLT_NULL. */ 1759 bpf_af = (uint32_t)af; 1760 bpf_ptap(ifp->if_bpf, m, &bpf_af, sizeof(bpf_af)); 1761 } 1762 bpf_reltoken(); 1763 } 1764 1765 static inline unsigned int 1766 calculate_padding(struct wg_packet *pkt) 1767 { 1768 unsigned int padded_size, last_unit; 1769 1770 last_unit = pkt->p_mbuf->m_pkthdr.len; 1771 1772 /* Keepalive packets don't set p_mtu, but also have a length of zero. */ 1773 if (__predict_false(pkt->p_mtu == 0)) 1774 return WG_PKT_WITH_PADDING(last_unit) - last_unit; 1775 1776 /* 1777 * Just in case the packet is bigger than the MTU and would cause 1778 * the final subtraction to overflow. 1779 */ 1780 if (__predict_false(last_unit > pkt->p_mtu)) 1781 last_unit %= pkt->p_mtu; 1782 1783 padded_size = MIN(pkt->p_mtu, WG_PKT_WITH_PADDING(last_unit)); 1784 return (padded_size - last_unit); 1785 } 1786 1787 static inline int 1788 determine_af_and_pullup(struct mbuf **m, sa_family_t *af) 1789 { 1790 const struct ip *ip; 1791 const struct ip6_hdr *ip6; 1792 int len; 1793 1794 ip = mtod(*m, const struct ip *); 1795 ip6 = mtod(*m, const struct ip6_hdr *); 1796 len = (*m)->m_pkthdr.len; 1797 1798 if (len >= sizeof(*ip) && ip->ip_v == IPVERSION) 1799 *af = AF_INET; 1800 #ifdef INET6 1801 else if (len >= sizeof(*ip6) && 1802 (ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1803 *af = AF_INET6; 1804 #endif 1805 else 1806 return (EAFNOSUPPORT); 1807 1808 *m = m_pullup(*m, (*af == AF_INET ? sizeof(*ip) : sizeof(*ip6))); 1809 if (*m == NULL) 1810 return (ENOBUFS); 1811 1812 return (0); 1813 } 1814 1815 static void 1816 wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt) 1817 { 1818 static const uint8_t padding[WG_PKT_PADDING] = { 0 }; 1819 struct wg_pkt_data *data; 1820 struct wg_peer *peer; 1821 struct noise_remote *remote; 1822 struct mbuf *m; 1823 unsigned int padlen, state = WG_PACKET_DEAD; 1824 uint32_t idx; 1825 1826 remote = noise_keypair_remote(pkt->p_keypair); 1827 peer = noise_remote_arg(remote); 1828 m = pkt->p_mbuf; 1829 1830 padlen = calculate_padding(pkt); 1831 if (padlen != 0 && !m_append(m, padlen, padding)) 1832 goto out; 1833 1834 if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_counter, m) != 0) 1835 goto out; 1836 1837 M_PREPEND(m, sizeof(struct wg_pkt_data), M_NOWAIT); 1838 if (m == NULL) 1839 goto out; 1840 data = mtod(m, struct wg_pkt_data *); 1841 data->t = WG_PKT_DATA; 1842 data->r_idx = idx; 1843 data->counter = htole64(pkt->p_counter); 1844 1845 state = WG_PACKET_CRYPTED; 1846 1847 out: 1848 pkt->p_mbuf = m; 1849 atomic_store_rel_int(&pkt->p_state, state); 1850 taskqueue_enqueue(peer->p_send_taskqueue, &peer->p_send_task); 1851 noise_remote_put(remote); 1852 } 1853 1854 static void 1855 wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt) 1856 { 1857 struct wg_peer *peer, *allowed_peer; 1858 struct noise_remote *remote; 1859 struct mbuf *m; 1860 unsigned int state = WG_PACKET_DEAD; 1861 int len; 1862 1863 remote = noise_keypair_remote(pkt->p_keypair); 1864 peer = noise_remote_arg(remote); 1865 m = pkt->p_mbuf; 1866 1867 pkt->p_counter = le64toh(mtod(m, struct wg_pkt_data *)->counter); 1868 m_adj(m, sizeof(struct wg_pkt_data)); 1869 1870 if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_counter, m) != 0) 1871 goto out; 1872 1873 /* A packet with a length of zero is a keepalive packet. */ 1874 if (__predict_false(m->m_pkthdr.len == 0)) { 1875 DPRINTF(sc, "Receiving keepalive packet from peer %ld\n", 1876 peer->p_id); 1877 state = WG_PACKET_CRYPTED; 1878 goto out; 1879 } 1880 1881 /* 1882 * Extract the source address for wg_aip_lookup(), and trim the 1883 * packet if it was padded before encryption. 1884 */ 1885 if (determine_af_and_pullup(&m, &pkt->p_af) != 0) 1886 goto out; 1887 if (pkt->p_af == AF_INET) { 1888 const struct ip *ip = mtod(m, const struct ip *); 1889 allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src); 1890 len = ntohs(ip->ip_len); 1891 if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) 1892 m_adj(m, len - m->m_pkthdr.len); 1893 } else { 1894 const struct ip6_hdr *ip6 = mtod(m, const struct ip6_hdr *); 1895 allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src); 1896 len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); 1897 if (len < m->m_pkthdr.len) 1898 m_adj(m, len - m->m_pkthdr.len); 1899 } 1900 1901 /* Drop the reference, since no need to dereference it. */ 1902 if (allowed_peer != NULL) 1903 noise_remote_put(allowed_peer->p_remote); 1904 1905 if (__predict_false(peer != allowed_peer)) { 1906 DPRINTF(sc, "Packet has disallowed src IP from peer %ld\n", 1907 peer->p_id); 1908 goto out; 1909 } 1910 1911 state = WG_PACKET_CRYPTED; 1912 1913 out: 1914 pkt->p_mbuf = m; 1915 atomic_store_rel_int(&pkt->p_state, state); 1916 taskqueue_enqueue(peer->p_recv_taskqueue, &peer->p_recv_task); 1917 noise_remote_put(remote); 1918 } 1919 1920 static void 1921 wg_encrypt_worker(void *arg, int pending __unused) 1922 { 1923 struct wg_softc *sc = arg; 1924 struct wg_queue *queue = &sc->sc_encrypt_parallel; 1925 struct wg_packet *pkt; 1926 1927 while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL) 1928 wg_encrypt(sc, pkt); 1929 } 1930 1931 static void 1932 wg_decrypt_worker(void *arg, int pending __unused) 1933 { 1934 struct wg_softc *sc = arg; 1935 struct wg_queue *queue = &sc->sc_decrypt_parallel; 1936 struct wg_packet *pkt; 1937 1938 while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL) 1939 wg_decrypt(sc, pkt); 1940 } 1941 1942 static void 1943 wg_encrypt_dispatch(struct wg_softc *sc) 1944 { 1945 int cpu; 1946 1947 /* 1948 * The update to encrypt_last_cpu is racy such that we may 1949 * reschedule the task for the same CPU multiple times, but 1950 * the race doesn't really matter. 1951 */ 1952 cpu = (sc->sc_encrypt_last_cpu + 1) % ncpus; 1953 sc->sc_encrypt_last_cpu = cpu; 1954 taskqueue_enqueue(wg_taskqueues[cpu], &sc->sc_encrypt_tasks[cpu]); 1955 } 1956 1957 static void 1958 wg_decrypt_dispatch(struct wg_softc *sc) 1959 { 1960 int cpu; 1961 1962 cpu = (sc->sc_decrypt_last_cpu + 1) % ncpus; 1963 sc->sc_decrypt_last_cpu = cpu; 1964 taskqueue_enqueue(wg_taskqueues[cpu], &sc->sc_decrypt_tasks[cpu]); 1965 } 1966 1967 static void 1968 wg_deliver_out(void *arg, int pending __unused) 1969 { 1970 struct wg_peer *peer = arg; 1971 struct wg_softc *sc = peer->p_sc; 1972 struct wg_queue *queue = &peer->p_encrypt_serial; 1973 struct wg_endpoint endpoint; 1974 struct wg_packet *pkt; 1975 struct mbuf *m; 1976 int len, cpu; 1977 1978 cpu = mycpuid; 1979 1980 while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) { 1981 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) { 1982 IFNET_STAT_INC(sc->sc_ifp, oerrors, 1); 1983 wg_packet_free(pkt); 1984 continue; 1985 } 1986 1987 m = pkt->p_mbuf; 1988 m->m_flags &= ~MBUF_CLEARFLAGS; 1989 len = m->m_pkthdr.len; 1990 1991 pkt->p_mbuf = NULL; 1992 wg_packet_free(pkt); 1993 1994 /* 1995 * The keepalive timers -- both persistent and mandatory -- 1996 * are part of the internal state machine, which needs to be 1997 * cranked whether or not the packet was actually sent. 1998 */ 1999 wg_timers_event_any_authenticated_packet_traversal(peer); 2000 wg_timers_event_any_authenticated_packet_sent(peer); 2001 2002 wg_peer_get_endpoint(peer, &endpoint); 2003 if (wg_send(sc, &endpoint, m) == 0) { 2004 peer->p_tx_bytes[cpu] += len; 2005 if (len > WG_PKT_ENCRYPTED_LEN(0)) 2006 wg_timers_event_data_sent(peer); 2007 if (noise_keypair_should_refresh(peer->p_remote, true)) 2008 wg_timers_event_want_initiation(peer); 2009 } 2010 } 2011 } 2012 2013 static void 2014 wg_deliver_in(void *arg, int pending __unused) 2015 { 2016 struct wg_peer *peer = arg; 2017 struct wg_softc *sc = peer->p_sc; 2018 struct wg_queue *queue = &peer->p_decrypt_serial; 2019 struct wg_packet *pkt; 2020 struct ifnet *ifp; 2021 struct mbuf *m; 2022 size_t rx_bytes; 2023 int cpu; 2024 2025 cpu = mycpuid; 2026 ifp = sc->sc_ifp; 2027 2028 while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) { 2029 if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED || 2030 noise_keypair_counter_check(pkt->p_keypair, pkt->p_counter) 2031 != 0) { 2032 IFNET_STAT_INC(ifp, ierrors, 1); 2033 wg_packet_free(pkt); 2034 continue; 2035 } 2036 2037 if (noise_keypair_received_with(pkt->p_keypair)) 2038 wg_timers_event_handshake_complete(peer); 2039 2040 wg_timers_event_any_authenticated_packet_received(peer); 2041 wg_timers_event_any_authenticated_packet_traversal(peer); 2042 wg_peer_set_endpoint(peer, &pkt->p_endpoint); 2043 2044 m = pkt->p_mbuf; 2045 rx_bytes = WG_PKT_ENCRYPTED_LEN(m->m_pkthdr.len); 2046 peer->p_rx_bytes[cpu] += rx_bytes; 2047 IFNET_STAT_INC(ifp, ipackets, 1); 2048 IFNET_STAT_INC(ifp, ibytes, rx_bytes); 2049 2050 if (m->m_pkthdr.len > 0) { 2051 if (ifp->if_capenable & IFCAP_RXCSUM) { 2052 /* 2053 * The packet is authentic as ensured by the 2054 * AEAD tag, so we can tell the networking 2055 * stack that this packet has valid checksums 2056 * and thus is unnecessary to check again. 2057 */ 2058 if (m->m_pkthdr.csum_flags & CSUM_IP) 2059 m->m_pkthdr.csum_flags |= 2060 (CSUM_IP_CHECKED | CSUM_IP_VALID); 2061 if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) { 2062 m->m_pkthdr.csum_flags |= 2063 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 2064 m->m_pkthdr.csum_data = 0xffff; 2065 } 2066 } 2067 m->m_flags &= ~MBUF_CLEARFLAGS; 2068 m->m_pkthdr.rcvif = ifp; 2069 2070 wg_bpf_ptap(ifp, m, pkt->p_af); 2071 2072 netisr_queue((pkt->p_af == AF_INET ? 2073 NETISR_IP : NETISR_IPV6), m); 2074 pkt->p_mbuf = NULL; 2075 2076 wg_timers_event_data_received(peer); 2077 } 2078 2079 wg_packet_free(pkt); 2080 2081 if (noise_keypair_should_refresh(peer->p_remote, false)) 2082 wg_timers_event_want_initiation(peer); 2083 } 2084 } 2085 2086 static void 2087 wg_input(struct wg_softc *sc, struct mbuf *m, const struct sockaddr *sa) 2088 { 2089 struct noise_remote *remote; 2090 struct wg_pkt_data *data; 2091 struct wg_packet *pkt; 2092 struct wg_peer *peer; 2093 struct mbuf *defragged; 2094 2095 /* 2096 * Defragment mbufs early on in order to: 2097 * - make the crypto a lot faster; 2098 * - make the subsequent m_pullup()'s no-ops. 2099 */ 2100 defragged = m_defrag(m, M_NOWAIT); 2101 if (defragged != NULL) 2102 m = defragged; /* The original mbuf chain is freed. */ 2103 2104 /* Ensure the packet is not shared before modifying it. */ 2105 m = m_unshare(m, M_NOWAIT); 2106 if (m == NULL) { 2107 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2108 return; 2109 } 2110 2111 /* Pullup enough to read packet type */ 2112 if ((m = m_pullup(m, sizeof(uint32_t))) == NULL) { 2113 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2114 return; 2115 } 2116 2117 if ((pkt = wg_packet_alloc(m)) == NULL) { 2118 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2119 m_freem(m); 2120 return; 2121 } 2122 2123 /* Save the remote address and port for later use. */ 2124 switch (sa->sa_family) { 2125 case AF_INET: 2126 pkt->p_endpoint.e_remote.r_sin = 2127 *(const struct sockaddr_in *)sa; 2128 break; 2129 #ifdef INET6 2130 case AF_INET6: 2131 pkt->p_endpoint.e_remote.r_sin6 = 2132 *(const struct sockaddr_in6 *)sa; 2133 break; 2134 #endif 2135 default: 2136 DPRINTF(sc, "Unsupported packet address family\n"); 2137 goto error; 2138 } 2139 2140 if (WG_PKT_IS_INITIATION(m) || 2141 WG_PKT_IS_RESPONSE(m) || 2142 WG_PKT_IS_COOKIE(m)) { 2143 if (!wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt)) { 2144 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2145 DPRINTF(sc, "Dropping handshake packet\n"); 2146 } 2147 taskqueue_enqueue(sc->sc_handshake_taskqueue, 2148 &sc->sc_handshake_task); 2149 return; 2150 } 2151 2152 if (WG_PKT_IS_DATA(m)) { 2153 /* Pullup the whole header to read r_idx below. */ 2154 pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data)); 2155 if (pkt->p_mbuf == NULL) 2156 goto error; 2157 2158 data = mtod(pkt->p_mbuf, struct wg_pkt_data *); 2159 pkt->p_keypair = noise_keypair_lookup(sc->sc_local, 2160 data->r_idx); 2161 if (pkt->p_keypair == NULL) 2162 goto error; 2163 2164 remote = noise_keypair_remote(pkt->p_keypair); 2165 peer = noise_remote_arg(remote); 2166 if (!wg_queue_both(&sc->sc_decrypt_parallel, 2167 &peer->p_decrypt_serial, pkt)) 2168 IFNET_STAT_INC(sc->sc_ifp, iqdrops, 1); 2169 2170 wg_decrypt_dispatch(sc); 2171 noise_remote_put(remote); 2172 return; 2173 } 2174 2175 error: 2176 IFNET_STAT_INC(sc->sc_ifp, ierrors, 1); 2177 wg_packet_free(pkt); 2178 } 2179 2180 static void 2181 wg_upcall(struct socket *so, void *arg, int waitflag __unused) 2182 { 2183 struct wg_softc *sc = arg; 2184 struct sockaddr *from; 2185 struct sockbuf sio; 2186 int ret, flags; 2187 2188 /* 2189 * For UDP, soreceive typically pulls just one packet, 2190 * so loop to get the whole batch. 2191 */ 2192 do { 2193 sbinit(&sio, 1000000000); /* really large to receive all */ 2194 flags = MSG_DONTWAIT; 2195 ret = so_pru_soreceive(so, &from, NULL, &sio, NULL, &flags); 2196 if (ret != 0 || sio.sb_mb == NULL) { 2197 if (from != NULL) 2198 kfree(from, M_SONAME); 2199 break; 2200 } 2201 wg_input(sc, sio.sb_mb, from); 2202 kfree(from, M_SONAME); 2203 } while (sio.sb_mb != NULL); 2204 } 2205 2206 static void 2207 wg_peer_send_staged(struct wg_peer *peer) 2208 { 2209 struct wg_softc *sc = peer->p_sc; 2210 struct wg_packet *pkt, *tpkt; 2211 struct wg_packet_list list; 2212 struct noise_keypair *keypair = NULL; 2213 2214 wg_queue_delist_staged(&peer->p_stage_queue, &list); 2215 2216 if (STAILQ_EMPTY(&list)) 2217 return; 2218 2219 if ((keypair = noise_keypair_current(peer->p_remote)) == NULL) 2220 goto error; 2221 2222 /* 2223 * We now try to assign counters to all of the packets in the queue. 2224 * If we can't assign counters for all of them, we just consider it 2225 * a failure and wait for the next handshake. 2226 */ 2227 STAILQ_FOREACH(pkt, &list, p_parallel) { 2228 if (!noise_keypair_counter_next(keypair, &pkt->p_counter)) 2229 goto error; 2230 } 2231 STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) { 2232 pkt->p_keypair = noise_keypair_ref(keypair); 2233 if (!wg_queue_both(&sc->sc_encrypt_parallel, 2234 &peer->p_encrypt_serial, pkt)) 2235 IFNET_STAT_INC(sc->sc_ifp, oqdrops, 1); 2236 } 2237 2238 wg_encrypt_dispatch(sc); 2239 noise_keypair_put(keypair); 2240 return; 2241 2242 error: 2243 if (keypair != NULL) 2244 noise_keypair_put(keypair); 2245 wg_queue_enlist_staged(&peer->p_stage_queue, &list); 2246 wg_timers_event_want_initiation(peer); 2247 } 2248 2249 static inline void 2250 xmit_err(struct ifnet *ifp, struct mbuf *m, struct wg_packet *pkt, 2251 sa_family_t af) 2252 { 2253 bool oerror = false; 2254 2255 if (m != NULL) { 2256 if (af == AF_INET) { 2257 icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0); 2258 oerror = true; 2259 #ifdef INET6 2260 } else if (af == AF_INET6) { 2261 icmp6_error(m, ICMP6_DST_UNREACH, 0, 0); 2262 oerror = true; 2263 #endif 2264 } else { 2265 m_freem(m); 2266 } 2267 2268 if (pkt != NULL) 2269 pkt->p_mbuf = NULL; 2270 } 2271 2272 if (pkt != NULL) 2273 wg_packet_free(pkt); 2274 2275 if (oerror) 2276 IFNET_STAT_INC(ifp, oerrors, 1); 2277 else 2278 IFNET_STAT_INC(ifp, oqdrops, 1); 2279 } 2280 2281 static int 2282 wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, 2283 struct rtentry *rt) 2284 { 2285 struct wg_softc *sc = ifp->if_softc; 2286 struct wg_packet *pkt = NULL; 2287 struct wg_peer *peer = NULL; 2288 struct mbuf *defragged; 2289 sa_family_t af = AF_UNSPEC; 2290 int ret; 2291 2292 if (dst->sa_family == AF_UNSPEC) { 2293 /* 2294 * Specially handle packets written/injected by BPF. 2295 * The packets have the same DLT_NULL link-layer type 2296 * (i.e., 4-byte link-layer header in host byte order). 2297 */ 2298 dst->sa_family = *(mtod(m, uint32_t *)); 2299 m_adj(m, sizeof(uint32_t)); 2300 } 2301 if (dst->sa_family == AF_UNSPEC) { 2302 ret = EAFNOSUPPORT; 2303 goto error; 2304 } 2305 2306 wg_bpf_ptap(ifp, m, dst->sa_family); 2307 2308 defragged = m_defrag(m, M_NOWAIT); 2309 if (defragged != NULL) 2310 m = defragged; 2311 2312 m = m_unshare(m, M_NOWAIT); 2313 if (m == NULL) { 2314 ret = ENOBUFS; 2315 goto error; 2316 } 2317 2318 if ((ret = determine_af_and_pullup(&m, &af)) != 0) 2319 goto error; 2320 if (af != dst->sa_family) { 2321 ret = EAFNOSUPPORT; 2322 goto error; 2323 } 2324 2325 if ((pkt = wg_packet_alloc(m)) == NULL) { 2326 ret = ENOBUFS; 2327 goto error; 2328 } 2329 2330 pkt->p_af = af; 2331 pkt->p_mtu = ifp->if_mtu; 2332 if (rt != NULL && rt->rt_rmx.rmx_mtu > 0 && 2333 rt->rt_rmx.rmx_mtu < pkt->p_mtu) 2334 pkt->p_mtu = rt->rt_rmx.rmx_mtu; 2335 2336 if (af == AF_INET) { 2337 peer = wg_aip_lookup(sc, AF_INET, 2338 &mtod(m, struct ip *)->ip_dst); 2339 } else { 2340 peer = wg_aip_lookup(sc, AF_INET6, 2341 &mtod(m, struct ip6_hdr *)->ip6_dst); 2342 } 2343 if (__predict_false(peer == NULL)) { 2344 ret = ENOKEY; 2345 goto error; 2346 } 2347 2348 if (__predict_false(peer->p_endpoint.e_remote.r_sa.sa_family 2349 == AF_UNSPEC)) { 2350 DPRINTF(sc, "No valid endpoint has been configured or " 2351 "discovered for peer %ld\n", peer->p_id); 2352 ret = EHOSTUNREACH; 2353 goto error; 2354 } 2355 2356 if (__predict_false(m->m_pkthdr.loop_cnt++ > MAX_LOOPS)) { 2357 DPRINTF(sc, "Packet looped"); 2358 ret = ELOOP; 2359 goto error; 2360 } 2361 2362 wg_queue_push_staged(&peer->p_stage_queue, pkt); 2363 wg_peer_send_staged(peer); 2364 noise_remote_put(peer->p_remote); 2365 2366 return (0); 2367 2368 error: 2369 if (peer != NULL) 2370 noise_remote_put(peer->p_remote); 2371 xmit_err(ifp, m, pkt, af); 2372 return (ret); 2373 } 2374 2375 /*----------------------------------------------------------------------------*/ 2376 /* Interface Functions */ 2377 2378 static int wg_up(struct wg_softc *); 2379 static void wg_down(struct wg_softc *); 2380 2381 static int 2382 wg_ioctl_get(struct wg_softc *sc, struct wg_data_io *data, bool privileged) 2383 { 2384 struct wg_interface_io *iface_p, iface_o; 2385 struct wg_peer_io *peer_p, peer_o; 2386 struct wg_aip_io *aip_p, aip_o; 2387 struct wg_peer *peer; 2388 struct wg_aip *aip; 2389 size_t size, peer_count, aip_count; 2390 int cpu, ret = 0; 2391 2392 lockmgr(&sc->sc_lock, LK_SHARED); 2393 2394 /* Determine the required data size. */ 2395 size = sizeof(struct wg_interface_io); 2396 size += sizeof(struct wg_peer_io) * sc->sc_peers_num; 2397 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) 2398 size += sizeof(struct wg_aip_io) * peer->p_aips_num; 2399 2400 /* Return the required size for userland allocation. */ 2401 if (data->wgd_size < size) { 2402 data->wgd_size = size; 2403 lockmgr(&sc->sc_lock, LK_RELEASE); 2404 return (0); 2405 } 2406 2407 iface_p = data->wgd_interface; 2408 bzero(&iface_o, sizeof(iface_o)); 2409 /* 2410 * No need to acquire the 'sc_socket.so_lock', because 'sc_lock' 2411 * is acquired and that's enough to prevent modifications to 2412 * 'sc_socket' members. 2413 */ 2414 if (sc->sc_socket.so_port != 0) { 2415 iface_o.i_port = sc->sc_socket.so_port; 2416 iface_o.i_flags |= WG_INTERFACE_HAS_PORT; 2417 } 2418 if (sc->sc_socket.so_user_cookie != 0) { 2419 iface_o.i_cookie = sc->sc_socket.so_user_cookie; 2420 iface_o.i_flags |= WG_INTERFACE_HAS_COOKIE; 2421 } 2422 if (noise_local_keys(sc->sc_local, iface_o.i_public, 2423 iface_o.i_private)) { 2424 iface_o.i_flags |= WG_INTERFACE_HAS_PUBLIC; 2425 if (privileged) 2426 iface_o.i_flags |= WG_INTERFACE_HAS_PRIVATE; 2427 else 2428 bzero(iface_o.i_private, sizeof(iface_o.i_private)); 2429 } 2430 2431 peer_count = 0; 2432 peer_p = &iface_p->i_peers[0]; 2433 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2434 bzero(&peer_o, sizeof(peer_o)); 2435 2436 peer_o.p_flags |= WG_PEER_HAS_PUBLIC; 2437 if (noise_remote_keys(peer->p_remote, peer_o.p_public, 2438 peer_o.p_psk)) { 2439 if (privileged) 2440 peer_o.p_flags |= WG_PEER_HAS_PSK; 2441 else 2442 bzero(peer_o.p_psk, sizeof(peer_o.p_psk)); 2443 } 2444 if (wg_timers_get_persistent_keepalive(peer, &peer_o.p_pka)) 2445 peer_o.p_flags |= WG_PEER_HAS_PKA; 2446 if (wg_peer_get_sockaddr(peer, &peer_o.p_sa) == 0) 2447 peer_o.p_flags |= WG_PEER_HAS_ENDPOINT; 2448 for (cpu = 0; cpu < ncpus; cpu++) { 2449 peer_o.p_rxbytes += peer->p_rx_bytes[cpu]; 2450 peer_o.p_txbytes += peer->p_tx_bytes[cpu]; 2451 } 2452 wg_timers_get_last_handshake(peer, &peer_o.p_last_handshake); 2453 peer_o.p_id = (uint64_t)peer->p_id; 2454 strlcpy(peer_o.p_description, peer->p_description, 2455 sizeof(peer_o.p_description)); 2456 2457 aip_count = 0; 2458 aip_p = &peer_p->p_aips[0]; 2459 LIST_FOREACH(aip, &peer->p_aips, a_entry) { 2460 bzero(&aip_o, sizeof(aip_o)); 2461 aip_o.a_af = aip->a_af; 2462 if (aip->a_af == AF_INET) { 2463 aip_o.a_cidr = bitcount32(aip->a_mask.ip); 2464 memcpy(&aip_o.a_ipv4, &aip->a_addr.in, 2465 sizeof(aip->a_addr.in)); 2466 } else if (aip->a_af == AF_INET6) { 2467 aip_o.a_cidr = in6_mask2len(&aip->a_mask.in6, 2468 NULL); 2469 memcpy(&aip_o.a_ipv6, &aip->a_addr.in6, 2470 sizeof(aip->a_addr.in6)); 2471 } 2472 2473 ret = copyout(&aip_o, aip_p, sizeof(aip_o)); 2474 if (ret != 0) 2475 goto out; 2476 2477 aip_p++; 2478 aip_count++; 2479 } 2480 KKASSERT(aip_count == peer->p_aips_num); 2481 peer_o.p_aips_count = aip_count; 2482 2483 ret = copyout(&peer_o, peer_p, sizeof(peer_o)); 2484 if (ret != 0) 2485 goto out; 2486 2487 peer_p = (struct wg_peer_io *)aip_p; 2488 peer_count++; 2489 } 2490 KKASSERT(peer_count == sc->sc_peers_num); 2491 iface_o.i_peers_count = peer_count; 2492 2493 ret = copyout(&iface_o, iface_p, sizeof(iface_o)); 2494 2495 out: 2496 lockmgr(&sc->sc_lock, LK_RELEASE); 2497 explicit_bzero(&iface_o, sizeof(iface_o)); 2498 explicit_bzero(&peer_o, sizeof(peer_o)); 2499 return (ret); 2500 } 2501 2502 static int 2503 wg_ioctl_set(struct wg_softc *sc, struct wg_data_io *data) 2504 { 2505 struct wg_interface_io *iface_p, iface_o; 2506 struct wg_peer_io *peer_p, peer_o; 2507 struct wg_aip_io *aip_p, aip_o; 2508 struct wg_peer *peer; 2509 struct noise_remote *remote; 2510 uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE]; 2511 size_t i, j; 2512 int ret; 2513 2514 remote = NULL; 2515 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2516 2517 iface_p = data->wgd_interface; 2518 if ((ret = copyin(iface_p, &iface_o, sizeof(iface_o))) != 0) 2519 goto error; 2520 2521 if (iface_o.i_flags & WG_INTERFACE_REPLACE_PEERS) 2522 wg_peer_destroy_all(sc); 2523 2524 if ((iface_o.i_flags & WG_INTERFACE_HAS_PRIVATE) && 2525 (!noise_local_keys(sc->sc_local, NULL, private) || 2526 timingsafe_bcmp(private, iface_o.i_private, WG_KEY_SIZE) != 0)) { 2527 if (curve25519_generate_public(public, iface_o.i_private)) { 2528 remote = noise_remote_lookup(sc->sc_local, public); 2529 if (remote != NULL) { 2530 /* Remove the conflicting peer. */ 2531 peer = noise_remote_arg(remote); 2532 wg_peer_destroy(peer); 2533 noise_remote_put(remote); 2534 } 2535 } 2536 2537 /* 2538 * Set the private key. 2539 * 2540 * Note: we might be removing the private key. 2541 */ 2542 if (noise_local_set_private(sc->sc_local, iface_o.i_private)) 2543 cookie_checker_update(sc->sc_cookie, public); 2544 else 2545 cookie_checker_update(sc->sc_cookie, NULL); 2546 } 2547 2548 if ((iface_o.i_flags & WG_INTERFACE_HAS_PORT) && 2549 iface_o.i_port != sc->sc_socket.so_port) { 2550 if (sc->sc_ifp->if_flags & IFF_RUNNING) { 2551 ret = wg_socket_init(sc, iface_o.i_port); 2552 if (ret != 0) 2553 goto error; 2554 } else { 2555 sc->sc_socket.so_port = iface_o.i_port; 2556 } 2557 } 2558 2559 if (iface_o.i_flags & WG_INTERFACE_HAS_COOKIE) { 2560 ret = wg_socket_set_cookie(sc, iface_o.i_cookie); 2561 if (ret != 0) 2562 goto error; 2563 } 2564 2565 peer_p = &iface_p->i_peers[0]; 2566 for (i = 0; i < iface_o.i_peers_count; i++) { 2567 if ((ret = copyin(peer_p, &peer_o, sizeof(peer_o))) != 0) 2568 goto error; 2569 2570 /* Peer must have public key. */ 2571 if ((peer_o.p_flags & WG_PEER_HAS_PUBLIC) == 0) 2572 goto next_peer; 2573 /* Ignore peer that has the same public key. */ 2574 if (noise_local_keys(sc->sc_local, public, NULL) && 2575 memcmp(public, peer_o.p_public, WG_KEY_SIZE) == 0) 2576 goto next_peer; 2577 2578 /* Lookup peer, or create if it doesn't exist. */ 2579 remote = noise_remote_lookup(sc->sc_local, peer_o.p_public); 2580 if (remote != NULL) { 2581 peer = noise_remote_arg(remote); 2582 } else { 2583 if (peer_o.p_flags & (WG_PEER_REMOVE | WG_PEER_UPDATE)) 2584 goto next_peer; 2585 2586 peer = wg_peer_create(sc, peer_o.p_public); 2587 if (peer == NULL) { 2588 ret = ENOMEM; 2589 goto error; 2590 } 2591 2592 /* No allowed IPs to remove for a new peer. */ 2593 peer_o.p_flags &= ~WG_PEER_REPLACE_AIPS; 2594 } 2595 2596 if (peer_o.p_flags & WG_PEER_REMOVE) { 2597 wg_peer_destroy(peer); 2598 goto next_peer; 2599 } 2600 2601 if (peer_o.p_flags & WG_PEER_HAS_ENDPOINT) { 2602 ret = wg_peer_set_sockaddr(peer, &peer_o.p_sa); 2603 if (ret != 0) 2604 goto error; 2605 } 2606 if (peer_o.p_flags & WG_PEER_HAS_PSK) 2607 noise_remote_set_psk(peer->p_remote, peer_o.p_psk); 2608 if (peer_o.p_flags & WG_PEER_HAS_PKA) 2609 wg_timers_set_persistent_keepalive(peer, peer_o.p_pka); 2610 if (peer_o.p_flags & WG_PEER_SET_DESCRIPTION) 2611 strlcpy(peer->p_description, peer_o.p_description, 2612 sizeof(peer->p_description)); 2613 2614 if (peer_o.p_flags & WG_PEER_REPLACE_AIPS) 2615 wg_aip_remove_all(sc, peer); 2616 2617 for (j = 0; j < peer_o.p_aips_count; j++) { 2618 aip_p = &peer_p->p_aips[j]; 2619 if ((ret = copyin(aip_p, &aip_o, sizeof(aip_o))) != 0) 2620 goto error; 2621 ret = wg_aip_add(sc, peer, aip_o.a_af, &aip_o.a_addr, 2622 aip_o.a_cidr); 2623 if (ret != 0) 2624 goto error; 2625 } 2626 2627 if (sc->sc_ifp->if_link_state == LINK_STATE_UP) 2628 wg_peer_send_staged(peer); 2629 2630 next_peer: 2631 if (remote != NULL) { 2632 noise_remote_put(remote); 2633 remote = NULL; 2634 } 2635 aip_p = &peer_p->p_aips[peer_o.p_aips_count]; 2636 peer_p = (struct wg_peer_io *)aip_p; 2637 } 2638 2639 error: 2640 if (remote != NULL) 2641 noise_remote_put(remote); 2642 lockmgr(&sc->sc_lock, LK_RELEASE); 2643 explicit_bzero(&iface_o, sizeof(iface_o)); 2644 explicit_bzero(&peer_o, sizeof(peer_o)); 2645 explicit_bzero(&aip_o, sizeof(aip_o)); 2646 explicit_bzero(public, sizeof(public)); 2647 explicit_bzero(private, sizeof(private)); 2648 return (ret); 2649 } 2650 2651 static int 2652 wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cred) 2653 { 2654 struct wg_data_io *wgd; 2655 struct wg_softc *sc; 2656 struct ifreq *ifr; 2657 bool privileged; 2658 int ret, mask; 2659 2660 sc = ifp->if_softc; 2661 ifr = (struct ifreq *)data; 2662 ret = 0; 2663 2664 switch (cmd) { 2665 case SIOCSWG: 2666 ret = caps_priv_check(cred, SYSCAP_RESTRICTEDROOT); 2667 if (ret == 0) { 2668 wgd = (struct wg_data_io *)data; 2669 ret = wg_ioctl_set(sc, wgd); 2670 } 2671 break; 2672 case SIOCGWG: 2673 privileged = 2674 (caps_priv_check(cred, SYSCAP_RESTRICTEDROOT) == 0); 2675 wgd = (struct wg_data_io *)data; 2676 ret = wg_ioctl_get(sc, wgd, privileged); 2677 break; 2678 /* Interface IOCTLs */ 2679 case SIOCSIFADDR: 2680 /* 2681 * This differs from *BSD norms, but is more uniform with how 2682 * WireGuard behaves elsewhere. 2683 */ 2684 break; 2685 case SIOCSIFFLAGS: 2686 if (ifp->if_flags & IFF_UP) 2687 ret = wg_up(sc); 2688 else 2689 wg_down(sc); 2690 break; 2691 case SIOCSIFMTU: 2692 if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU) 2693 ret = EINVAL; 2694 else 2695 ifp->if_mtu = ifr->ifr_mtu; 2696 break; 2697 case SIOCSIFCAP: 2698 mask = ifp->if_capenable ^ ifr->ifr_reqcap; 2699 if (mask & IFCAP_RXCSUM) 2700 ifp->if_capenable ^= IFCAP_RXCSUM; 2701 break; 2702 case SIOCADDMULTI: 2703 case SIOCDELMULTI: 2704 break; 2705 default: 2706 ret = ENOTTY; 2707 } 2708 2709 return (ret); 2710 } 2711 2712 static int 2713 wg_up(struct wg_softc *sc) 2714 { 2715 struct ifnet *ifp = sc->sc_ifp; 2716 struct wg_peer *peer; 2717 int ret = 0; 2718 2719 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2720 2721 /* Silent success if we're already running. */ 2722 if (ifp->if_flags & IFF_RUNNING) 2723 goto out; 2724 ifp->if_flags |= IFF_RUNNING; 2725 2726 ret = wg_socket_init(sc, sc->sc_socket.so_port); 2727 if (ret == 0) { 2728 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) 2729 wg_timers_enable(peer); 2730 ifp->if_link_state = LINK_STATE_UP; 2731 if_link_state_change(ifp); 2732 } else { 2733 ifp->if_flags &= ~IFF_RUNNING; 2734 DPRINTF(sc, "Unable to initialize sockets: %d\n", ret); 2735 } 2736 2737 out: 2738 lockmgr(&sc->sc_lock, LK_RELEASE); 2739 return (ret); 2740 } 2741 2742 static void 2743 wg_down(struct wg_softc *sc) 2744 { 2745 struct ifnet *ifp = sc->sc_ifp; 2746 struct wg_peer *peer; 2747 int i; 2748 2749 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2750 2751 if ((ifp->if_flags & IFF_RUNNING) == 0) { 2752 lockmgr(&sc->sc_lock, LK_RELEASE); 2753 return; 2754 } 2755 ifp->if_flags &= ~IFF_RUNNING; 2756 2757 /* Cancel all tasks. */ 2758 while (taskqueue_cancel(sc->sc_handshake_taskqueue, 2759 &sc->sc_handshake_task, NULL) != 0) { 2760 taskqueue_drain(sc->sc_handshake_taskqueue, 2761 &sc->sc_handshake_task); 2762 } 2763 for (i = 0; i < ncpus; i++) { 2764 while (taskqueue_cancel(wg_taskqueues[i], 2765 &sc->sc_encrypt_tasks[i], NULL) != 0) { 2766 taskqueue_drain(wg_taskqueues[i], 2767 &sc->sc_encrypt_tasks[i]); 2768 } 2769 while (taskqueue_cancel(wg_taskqueues[i], 2770 &sc->sc_decrypt_tasks[i], NULL) != 0) { 2771 taskqueue_drain(wg_taskqueues[i], 2772 &sc->sc_decrypt_tasks[i]); 2773 } 2774 } 2775 2776 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2777 wg_queue_purge(&peer->p_stage_queue); 2778 wg_timers_disable(peer); 2779 } 2780 2781 wg_queue_purge(&sc->sc_handshake_queue); 2782 2783 TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) { 2784 noise_remote_handshake_clear(peer->p_remote); 2785 noise_remote_keypairs_clear(peer->p_remote); 2786 } 2787 2788 ifp->if_link_state = LINK_STATE_DOWN; 2789 if_link_state_change(ifp); 2790 wg_socket_uninit(sc); 2791 2792 lockmgr(&sc->sc_lock, LK_RELEASE); 2793 } 2794 2795 static int 2796 wg_clone_create(struct if_clone *ifc __unused, int unit, 2797 caddr_t params __unused, caddr_t data __unused) 2798 { 2799 struct wg_softc *sc; 2800 struct ifnet *ifp; 2801 int i; 2802 2803 sc = kmalloc(sizeof(*sc), M_WG, M_WAITOK | M_ZERO); 2804 2805 if (!rn_inithead(&sc->sc_aip4, wg_maskhead, 2806 offsetof(struct aip_addr, in)) || 2807 !rn_inithead(&sc->sc_aip6, wg_maskhead, 2808 offsetof(struct aip_addr, in6))) { 2809 if (sc->sc_aip4 != NULL) 2810 rn_freehead(sc->sc_aip4); 2811 if (sc->sc_aip6 != NULL) 2812 rn_freehead(sc->sc_aip6); 2813 kfree(sc, M_WG); 2814 return (ENOMEM); 2815 } 2816 2817 lockinit(&sc->sc_lock, "wg softc lock", 0, 0); 2818 lockinit(&sc->sc_aip_lock, "wg aip lock", 0, 0); 2819 2820 sc->sc_local = noise_local_alloc(); 2821 sc->sc_cookie = cookie_checker_alloc(); 2822 2823 TAILQ_INIT(&sc->sc_peers); 2824 2825 sc->sc_handshake_taskqueue = wg_taskqueues[karc4random() % ncpus]; 2826 TASK_INIT(&sc->sc_handshake_task, 0, wg_handshake_worker, sc); 2827 wg_queue_init(&sc->sc_handshake_queue, "hsq"); 2828 2829 sc->sc_encrypt_tasks = kmalloc(sizeof(*sc->sc_encrypt_tasks) * ncpus, 2830 M_WG, M_WAITOK | M_ZERO); 2831 sc->sc_decrypt_tasks = kmalloc(sizeof(*sc->sc_decrypt_tasks) * ncpus, 2832 M_WG, M_WAITOK | M_ZERO); 2833 for (i = 0; i < ncpus; i++) { 2834 TASK_INIT(&sc->sc_encrypt_tasks[i], 0, wg_encrypt_worker, sc); 2835 TASK_INIT(&sc->sc_decrypt_tasks[i], 0, wg_decrypt_worker, sc); 2836 } 2837 wg_queue_init(&sc->sc_encrypt_parallel, "encp"); 2838 wg_queue_init(&sc->sc_decrypt_parallel, "decp"); 2839 2840 ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD); 2841 if_initname(ifp, wgname, unit); 2842 ifp->if_softc = sc; 2843 ifp->if_mtu = DEFAULT_MTU; 2844 ifp->if_flags = IFF_NOARP | IFF_MULTICAST; 2845 ifp->if_capabilities = ifp->if_capenable = IFCAP_RXCSUM; 2846 ifp->if_output = wg_output; 2847 ifp->if_ioctl = wg_ioctl; 2848 ifq_set_maxlen(&ifp->if_snd, ifqmaxlen); 2849 ifq_set_ready(&ifp->if_snd); 2850 2851 if_attach(ifp, NULL); 2852 2853 /* DLT_NULL link-layer header: a 4-byte field in host byte order */ 2854 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2855 2856 lockmgr(&wg_mtx, LK_EXCLUSIVE); 2857 LIST_INSERT_HEAD(&wg_list, sc, sc_entry); 2858 lockmgr(&wg_mtx, LK_RELEASE); 2859 2860 return (0); 2861 } 2862 2863 static int 2864 wg_clone_destroy(struct ifnet *ifp) 2865 { 2866 struct wg_softc *sc = ifp->if_softc; 2867 2868 wg_down(sc); 2869 2870 lockmgr(&sc->sc_lock, LK_EXCLUSIVE); 2871 2872 kfree(sc->sc_encrypt_tasks, M_WG); 2873 kfree(sc->sc_decrypt_tasks, M_WG); 2874 wg_queue_deinit(&sc->sc_handshake_queue); 2875 wg_queue_deinit(&sc->sc_encrypt_parallel); 2876 wg_queue_deinit(&sc->sc_decrypt_parallel); 2877 2878 wg_peer_destroy_all(sc); 2879 2880 /* 2881 * Detach and free the interface before the sc_aip4 and sc_aip6 radix 2882 * trees, because the purge of interface's IPv6 addresses can cause 2883 * packet transmission and thus wg_aip_lookup() calls. 2884 */ 2885 bpfdetach(ifp); 2886 if_detach(ifp); 2887 if_free(ifp); 2888 2889 /* 2890 * All peers have been removed, so the sc_aip4 and sc_aip6 radix trees 2891 * must be empty now. 2892 */ 2893 rn_freehead(sc->sc_aip4); 2894 rn_freehead(sc->sc_aip6); 2895 lockuninit(&sc->sc_aip_lock); 2896 2897 cookie_checker_free(sc->sc_cookie); 2898 noise_local_free(sc->sc_local); 2899 2900 lockmgr(&wg_mtx, LK_EXCLUSIVE); 2901 LIST_REMOVE(sc, sc_entry); 2902 lockmgr(&wg_mtx, LK_RELEASE); 2903 2904 lockmgr(&sc->sc_lock, LK_RELEASE); 2905 lockuninit(&sc->sc_lock); 2906 kfree(sc, M_WG); 2907 2908 return (0); 2909 } 2910 2911 /*----------------------------------------------------------------------------*/ 2912 /* Module Interface */ 2913 2914 #ifdef WG_SELFTESTS 2915 #include "selftest/allowedips.c" 2916 static bool 2917 wg_run_selftests(void) 2918 { 2919 bool ret = true; 2920 2921 ret &= wg_allowedips_selftest(); 2922 ret &= noise_counter_selftest(); 2923 ret &= cookie_selftest(); 2924 2925 kprintf("%s: %s\n", __func__, ret ? "pass" : "FAIL"); 2926 return (ret); 2927 } 2928 #else /* !WG_SELFTESTS */ 2929 static inline bool 2930 wg_run_selftests(void) 2931 { 2932 return (true); 2933 } 2934 #endif /* WG_SELFTESTS */ 2935 2936 static struct if_clone wg_cloner = IF_CLONE_INITIALIZER( 2937 wgname, wg_clone_create, wg_clone_destroy, 0, IF_MAXUNIT); 2938 2939 static int 2940 wg_module_init(void) 2941 { 2942 int i, ret; 2943 2944 lockinit(&wg_mtx, "wg mtx lock", 0, 0); 2945 2946 wg_packet_zone = objcache_create_simple(M_WG_PACKET, 2947 sizeof(struct wg_packet)); 2948 if (wg_packet_zone == NULL) 2949 return (ENOMEM); 2950 2951 wg_taskqueues = kmalloc(sizeof(*wg_taskqueues) * ncpus, M_WG, 2952 M_WAITOK | M_ZERO); 2953 for (i = 0; i < ncpus; i++) { 2954 wg_taskqueues[i] = taskqueue_create("wg_taskq", M_WAITOK, 2955 taskqueue_thread_enqueue, 2956 &wg_taskqueues[i]); 2957 taskqueue_start_threads(&wg_taskqueues[i], 1, 2958 TDPRI_KERN_DAEMON, i, 2959 "wg_taskq_cpu_%d", i); 2960 } 2961 2962 if (!rn_inithead(&wg_maskhead, NULL, 0)) 2963 return (ENOMEM); 2964 2965 ret = cookie_init(); 2966 if (ret != 0) 2967 return (ret); 2968 ret = noise_init(); 2969 if (ret != 0) 2970 return (ret); 2971 2972 ret = if_clone_attach(&wg_cloner); 2973 if (ret != 0) 2974 return (ret); 2975 2976 if (!wg_run_selftests()) 2977 return (ENOTRECOVERABLE); 2978 2979 return (0); 2980 } 2981 2982 static int 2983 wg_module_deinit(void) 2984 { 2985 int i; 2986 2987 lockmgr(&wg_mtx, LK_EXCLUSIVE); 2988 2989 if (!LIST_EMPTY(&wg_list)) { 2990 lockmgr(&wg_mtx, LK_RELEASE); 2991 return (EBUSY); 2992 } 2993 2994 if_clone_detach(&wg_cloner); 2995 2996 noise_deinit(); 2997 cookie_deinit(); 2998 2999 for (i = 0; i < ncpus; i++) 3000 taskqueue_free(wg_taskqueues[i]); 3001 kfree(wg_taskqueues, M_WG); 3002 3003 rn_flush(wg_maskhead, rn_freemask); 3004 rn_freehead(wg_maskhead); 3005 3006 if (wg_packet_zone != NULL) 3007 objcache_destroy(wg_packet_zone); 3008 3009 lockmgr(&wg_mtx, LK_RELEASE); 3010 lockuninit(&wg_mtx); 3011 3012 return (0); 3013 } 3014 3015 static int 3016 wg_module_event_handler(module_t mod __unused, int what, void *arg __unused) 3017 { 3018 switch (what) { 3019 case MOD_LOAD: 3020 return wg_module_init(); 3021 case MOD_UNLOAD: 3022 return wg_module_deinit(); 3023 default: 3024 return (EOPNOTSUPP); 3025 } 3026 } 3027 3028 static moduledata_t wg_moduledata = { 3029 "if_wg", 3030 wg_module_event_handler, 3031 NULL 3032 }; 3033 3034 DECLARE_MODULE(if_wg, wg_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); 3035 MODULE_VERSION(if_wg, 1); /* WireGuard version */ 3036 MODULE_DEPEND(if_wg, crypto, 1, 1, 1); 3037