1 /* $NetBSD: if_wg.c,v 1.51 2020/08/31 20:34:43 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) Ryota Ozaki <ozaki.ryota@gmail.com> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the project nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * This network interface aims to implement the WireGuard protocol. 34 * The implementation is based on the paper of WireGuard as of 35 * 2018-06-30 [1]. The paper is referred in the source code with label 36 * [W]. Also the specification of the Noise protocol framework as of 37 * 2018-07-11 [2] is referred with label [N]. 38 * 39 * [1] https://www.wireguard.com/papers/wireguard.pdf 40 * [2] http://noiseprotocol.org/noise.pdf 41 */ 42 43 #include <sys/cdefs.h> 44 __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.51 2020/08/31 20:34:43 riastradh Exp $"); 45 46 #ifdef _KERNEL_OPT 47 #include "opt_inet.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/types.h> 52 53 #include <sys/atomic.h> 54 #include <sys/callout.h> 55 #include <sys/cprng.h> 56 #include <sys/cpu.h> 57 #include <sys/device.h> 58 #include <sys/domain.h> 59 #include <sys/errno.h> 60 #include <sys/intr.h> 61 #include <sys/ioctl.h> 62 #include <sys/kernel.h> 63 #include <sys/kmem.h> 64 #include <sys/kthread.h> 65 #include <sys/mbuf.h> 66 #include <sys/module.h> 67 #include <sys/mutex.h> 68 #include <sys/pcq.h> 69 #include <sys/percpu.h> 70 #include <sys/pserialize.h> 71 #include <sys/psref.h> 72 #include <sys/queue.h> 73 #include <sys/rwlock.h> 74 #include <sys/socket.h> 75 #include <sys/socketvar.h> 76 #include <sys/sockio.h> 77 #include <sys/sysctl.h> 78 #include <sys/syslog.h> 79 #include <sys/systm.h> 80 #include <sys/thmap.h> 81 #include <sys/time.h> 82 #include <sys/timespec.h> 83 84 #include <net/bpf.h> 85 #include <net/if.h> 86 #include <net/if_types.h> 87 #include <net/if_wg.h> 88 #include <net/route.h> 89 90 #include <netinet/in.h> 91 #include <netinet/in_pcb.h> 92 #include <netinet/in_var.h> 93 #include <netinet/ip.h> 94 #include <netinet/ip_var.h> 95 #include <netinet/udp.h> 96 #include <netinet/udp_var.h> 97 98 #ifdef INET6 99 #include <netinet/ip6.h> 100 #include <netinet6/in6_pcb.h> 101 #include <netinet6/in6_var.h> 102 #include <netinet6/ip6_var.h> 103 #include <netinet6/udp6_var.h> 104 #endif /* INET6 */ 105 106 #include <prop/proplib.h> 107 108 #include <crypto/blake2/blake2s.h> 109 #include <crypto/sodium/crypto_aead_chacha20poly1305.h> 110 #include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 111 #include <crypto/sodium/crypto_scalarmult.h> 112 113 #include "ioconf.h" 114 115 #ifdef WG_RUMPKERNEL 116 #include "wg_user.h" 117 #endif 118 119 /* 120 * Data structures 121 * - struct wg_softc is an instance of wg interfaces 122 * - It has a list of peers (struct wg_peer) 123 * - It has a kthread that sends/receives handshake messages and 124 * runs event handlers 125 * - It has its own two routing tables: one is for IPv4 and the other IPv6 126 * - struct wg_peer is a representative of a peer 127 * - It has a softint that is used to send packets over an wg interface 128 * to a peer 129 * - It has a pair of session instances (struct wg_session) 130 * - It has a pair of endpoint instances (struct wg_sockaddr) 131 * - Normally one endpoint is used and the second one is used only on 132 * a peer migration (a change of peer's IP address) 133 * - It has a list of IP addresses and sub networks called allowedips 134 * (struct wg_allowedip) 135 * - A packets sent over a session is allowed if its destination matches 136 * any IP addresses or sub networks of the list 137 * - struct wg_session represents a session of a secure tunnel with a peer 138 * - Two instances of sessions belong to a peer; a stable session and a 139 * unstable session 140 * - A handshake process of a session always starts with a unstable instance 141 * - Once a session is established, its instance becomes stable and the 142 * other becomes unstable instead 143 * - Data messages are always sent via a stable session 144 * 145 * Locking notes: 146 * - wg interfaces (struct wg_softc, wg) is listed in wg_softcs.list and 147 * protected by wg_softcs.lock 148 * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock 149 * - Changes to the peer list are serialized by wg_lock 150 * - The peer list may be read with pserialize(9) and psref(9) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 152 * => XXX replace by pserialize when routing table is psz-safe 153 * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken 154 * only in thread context and serializes: 155 * - the stable and unstable session pointers 156 * - all unstable session state 157 * - Packet processing may be done in softint context: 158 * - The stable session can be read under pserialize(9) or psref(9) 159 * - The stable session is always ESTABLISHED 160 * - On a session swap, we must wait for all readers to release a 161 * reference to a stable session before changing wgs_state and 162 * session states 163 * - Lock order: wg_lock -> wgp_lock 164 */ 165 166 167 #define WGLOG(level, fmt, args...) \ 168 log(level, "%s: " fmt, __func__, ##args) 169 170 /* Debug options */ 171 #ifdef WG_DEBUG 172 /* Output debug logs */ 173 #ifndef WG_DEBUG_LOG 174 #define WG_DEBUG_LOG 175 #endif 176 /* Output trace logs */ 177 #ifndef WG_DEBUG_TRACE 178 #define WG_DEBUG_TRACE 179 #endif 180 /* Output hash values, etc. */ 181 #ifndef WG_DEBUG_DUMP 182 #define WG_DEBUG_DUMP 183 #endif 184 /* Make some internal parameters configurable for testing and debugging */ 185 #ifndef WG_DEBUG_PARAMS 186 #define WG_DEBUG_PARAMS 187 #endif 188 #endif 189 190 #ifdef WG_DEBUG_TRACE 191 #define WG_TRACE(msg) \ 192 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)) 193 #else 194 #define WG_TRACE(msg) __nothing 195 #endif 196 197 #ifdef WG_DEBUG_LOG 198 #define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args) 199 #else 200 #define WG_DLOG(fmt, args...) __nothing 201 #endif 202 203 #define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 204 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 205 &(wgprc)->wgprc_curpps, 1)) { \ 206 log(level, fmt, ##args); \ 207 } \ 208 } while (0) 209 210 #ifdef WG_DEBUG_PARAMS 211 static bool wg_force_underload = false; 212 #endif 213 214 #ifdef WG_DEBUG_DUMP 215 216 #ifdef WG_RUMPKERNEL 217 static void 218 wg_dump_buf(const char *func, const char *buf, const size_t size) 219 { 220 221 log(LOG_DEBUG, "%s: ", func); 222 for (int i = 0; i < size; i++) 223 log(LOG_DEBUG, "%02x ", (int)(0xff & buf[i])); 224 log(LOG_DEBUG, "\n"); 225 } 226 #endif 227 228 static void 229 wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 230 const size_t size) 231 { 232 233 log(LOG_DEBUG, "%s: %s: ", func, name); 234 for (int i = 0; i < size; i++) 235 log(LOG_DEBUG, "%02x ", (int)(0xff & hash[i])); 236 log(LOG_DEBUG, "\n"); 237 } 238 239 #define WG_DUMP_HASH(name, hash) \ 240 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 241 #define WG_DUMP_HASH48(name, hash) \ 242 wg_dump_hash(__func__, name, hash, 48) 243 #define WG_DUMP_BUF(buf, size) \ 244 wg_dump_buf(__func__, buf, size) 245 #else 246 #define WG_DUMP_HASH(name, hash) __nothing 247 #define WG_DUMP_HASH48(name, hash) __nothing 248 #define WG_DUMP_BUF(buf, size) __nothing 249 #endif /* WG_DEBUG_DUMP */ 250 251 #define WG_MTU 1420 252 #define WG_ALLOWEDIPS 16 253 254 #define CURVE25519_KEY_LEN 32 255 #define TAI64N_LEN sizeof(uint32_t) * 3 256 #define POLY1305_AUTHTAG_LEN 16 257 #define HMAC_BLOCK_LEN 64 258 259 /* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 260 /* [N] 4.3: Hash functions */ 261 #define NOISE_DHLEN 32 262 /* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 263 #define NOISE_HASHLEN 32 264 #define NOISE_BLOCKLEN 64 265 #define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 266 /* [N] 5.1: "k" */ 267 #define NOISE_CIPHER_KEY_LEN 32 268 /* 269 * [N] 9.2: "psk" 270 * "... psk is a 32-byte secret value provided by the application." 271 */ 272 #define NOISE_PRESHARED_KEY_LEN 32 273 274 #define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 275 #define WG_TIMESTAMP_LEN TAI64N_LEN 276 277 #define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 278 279 #define WG_COOKIE_LEN 16 280 #define WG_MAC_LEN 16 281 #define WG_RANDVAL_LEN 24 282 283 #define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 284 /* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 285 #define WG_CHAINING_KEY_LEN NOISE_HASHLEN 286 /* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 287 #define WG_HASH_LEN NOISE_HASHLEN 288 #define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 289 #define WG_DH_OUTPUT_LEN NOISE_DHLEN 290 #define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 291 #define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 292 #define WG_DATA_KEY_LEN 32 293 #define WG_SALT_LEN 24 294 295 /* 296 * The protocol messages 297 */ 298 struct wg_msg { 299 uint32_t wgm_type; 300 } __packed; 301 302 /* [W] 5.4.2 First Message: Initiator to Responder */ 303 struct wg_msg_init { 304 uint32_t wgmi_type; 305 uint32_t wgmi_sender; 306 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 307 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 308 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 309 uint8_t wgmi_mac1[WG_MAC_LEN]; 310 uint8_t wgmi_mac2[WG_MAC_LEN]; 311 } __packed; 312 313 /* [W] 5.4.3 Second Message: Responder to Initiator */ 314 struct wg_msg_resp { 315 uint32_t wgmr_type; 316 uint32_t wgmr_sender; 317 uint32_t wgmr_receiver; 318 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 319 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 320 uint8_t wgmr_mac1[WG_MAC_LEN]; 321 uint8_t wgmr_mac2[WG_MAC_LEN]; 322 } __packed; 323 324 /* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 325 struct wg_msg_data { 326 uint32_t wgmd_type; 327 uint32_t wgmd_receiver; 328 uint64_t wgmd_counter; 329 uint32_t wgmd_packet[0]; 330 } __packed; 331 332 /* [W] 5.4.7 Under Load: Cookie Reply Message */ 333 struct wg_msg_cookie { 334 uint32_t wgmc_type; 335 uint32_t wgmc_receiver; 336 uint8_t wgmc_salt[WG_SALT_LEN]; 337 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 338 } __packed; 339 340 #define WG_MSG_TYPE_INIT 1 341 #define WG_MSG_TYPE_RESP 2 342 #define WG_MSG_TYPE_COOKIE 3 343 #define WG_MSG_TYPE_DATA 4 344 #define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 345 346 /* Sliding windows */ 347 348 #define SLIWIN_BITS 2048u 349 #define SLIWIN_TYPE uint32_t 350 #define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 351 #define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 352 #define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 353 354 struct sliwin { 355 SLIWIN_TYPE B[SLIWIN_WORDS]; 356 uint64_t T; 357 }; 358 359 static void 360 sliwin_reset(struct sliwin *W) 361 { 362 363 memset(W, 0, sizeof(*W)); 364 } 365 366 static int 367 sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 368 { 369 370 /* 371 * If it's more than one window older than the highest sequence 372 * number we've seen, reject. 373 */ 374 #ifdef __HAVE_ATOMIC64_LOADSTORE 375 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 376 return EAUTH; 377 #endif 378 379 /* 380 * Otherwise, we need to take the lock to decide, so don't 381 * reject just yet. Caller must serialize a call to 382 * sliwin_update in this case. 383 */ 384 return 0; 385 } 386 387 static int 388 sliwin_update(struct sliwin *W, uint64_t S) 389 { 390 unsigned word, bit; 391 392 /* 393 * If it's more than one window older than the highest sequence 394 * number we've seen, reject. 395 */ 396 if (S + SLIWIN_NPKT < W->T) 397 return EAUTH; 398 399 /* 400 * If it's higher than the highest sequence number we've seen, 401 * advance the window. 402 */ 403 if (S > W->T) { 404 uint64_t i = W->T / SLIWIN_BPW; 405 uint64_t j = S / SLIWIN_BPW; 406 unsigned k; 407 408 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 409 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 410 #ifdef __HAVE_ATOMIC64_LOADSTORE 411 atomic_store_relaxed(&W->T, S); 412 #else 413 W->T = S; 414 #endif 415 } 416 417 /* Test and set the bit -- if already set, reject. */ 418 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 419 bit = S % SLIWIN_BPW; 420 if (W->B[word] & (1UL << bit)) 421 return EAUTH; 422 W->B[word] |= 1UL << bit; 423 424 /* Accept! */ 425 return 0; 426 } 427 428 struct wg_worker { 429 kmutex_t wgw_lock; 430 kcondvar_t wgw_cv; 431 bool wgw_todie; 432 struct socket *wgw_so4; 433 struct socket *wgw_so6; 434 int wgw_wakeup_reasons; 435 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 __BIT(0) 436 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6 __BIT(1) 437 #define WG_WAKEUP_REASON_PEER __BIT(2) 438 }; 439 440 struct wg_session { 441 struct wg_peer *wgs_peer; 442 struct psref_target 443 wgs_psref; 444 445 int wgs_state; 446 #define WGS_STATE_UNKNOWN 0 447 #define WGS_STATE_INIT_ACTIVE 1 448 #define WGS_STATE_INIT_PASSIVE 2 449 #define WGS_STATE_ESTABLISHED 3 450 #define WGS_STATE_DESTROYING 4 451 452 time_t wgs_time_established; 453 time_t wgs_time_last_data_sent; 454 bool wgs_is_initiator; 455 456 uint32_t wgs_local_index; 457 uint32_t wgs_remote_index; 458 #ifdef __HAVE_ATOMIC64_LOADSTORE 459 volatile uint64_t 460 wgs_send_counter; 461 #else 462 kmutex_t wgs_send_counter_lock; 463 uint64_t wgs_send_counter; 464 #endif 465 466 struct { 467 kmutex_t lock; 468 struct sliwin window; 469 } *wgs_recvwin; 470 471 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 472 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 473 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 474 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 475 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 476 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 477 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 478 }; 479 480 struct wg_sockaddr { 481 union { 482 struct sockaddr_storage _ss; 483 struct sockaddr _sa; 484 struct sockaddr_in _sin; 485 struct sockaddr_in6 _sin6; 486 }; 487 struct psref_target wgsa_psref; 488 }; 489 490 #define wgsatoss(wgsa) (&(wgsa)->_ss) 491 #define wgsatosa(wgsa) (&(wgsa)->_sa) 492 #define wgsatosin(wgsa) (&(wgsa)->_sin) 493 #define wgsatosin6(wgsa) (&(wgsa)->_sin6) 494 495 #define wgsa_family(wgsa) (wgsatosa(wgsa)->sa_family) 496 497 struct wg_peer; 498 struct wg_allowedip { 499 struct radix_node wga_nodes[2]; 500 struct wg_sockaddr _wga_sa_addr; 501 struct wg_sockaddr _wga_sa_mask; 502 #define wga_sa_addr _wga_sa_addr._sa 503 #define wga_sa_mask _wga_sa_mask._sa 504 505 int wga_family; 506 uint8_t wga_cidr; 507 union { 508 struct in_addr _ip4; 509 struct in6_addr _ip6; 510 } wga_addr; 511 #define wga_addr4 wga_addr._ip4 512 #define wga_addr6 wga_addr._ip6 513 514 struct wg_peer *wga_peer; 515 }; 516 517 typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 518 519 struct wg_ppsratecheck { 520 struct timeval wgprc_lasttime; 521 int wgprc_curpps; 522 }; 523 524 struct wg_softc; 525 struct wg_peer { 526 struct wg_softc *wgp_sc; 527 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 528 struct pslist_entry wgp_peerlist_entry; 529 pserialize_t wgp_psz; 530 struct psref_target wgp_psref; 531 kmutex_t *wgp_lock; 532 533 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 534 struct wg_sockaddr *wgp_endpoint; 535 struct wg_sockaddr *wgp_endpoint0; 536 volatile unsigned wgp_endpoint_changing; 537 bool wgp_endpoint_available; 538 539 /* The preshared key (optional) */ 540 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 541 542 void *wgp_si; 543 pcq_t *wgp_q; 544 545 struct wg_session *wgp_session_stable; 546 struct wg_session *wgp_session_unstable; 547 548 /* timestamp in big-endian */ 549 wg_timestamp_t wgp_timestamp_latest_init; 550 551 struct timespec wgp_last_handshake_time; 552 553 callout_t wgp_rekey_timer; 554 callout_t wgp_handshake_timeout_timer; 555 callout_t wgp_session_dtor_timer; 556 557 time_t wgp_handshake_start_time; 558 559 int wgp_n_allowedips; 560 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 561 562 time_t wgp_latest_cookie_time; 563 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 564 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 565 bool wgp_last_sent_mac1_valid; 566 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 567 bool wgp_last_sent_cookie_valid; 568 569 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 570 571 time_t wgp_last_genrandval_time; 572 uint32_t wgp_randval; 573 574 struct wg_ppsratecheck wgp_ppsratecheck; 575 576 volatile unsigned int wgp_tasks; 577 #define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 578 #define WGP_TASK_RETRY_HANDSHAKE __BIT(1) 579 #define WGP_TASK_ESTABLISH_SESSION __BIT(2) 580 #define WGP_TASK_ENDPOINT_CHANGED __BIT(3) 581 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(4) 582 #define WGP_TASK_DESTROY_PREV_SESSION __BIT(5) 583 }; 584 585 struct wg_ops; 586 587 struct wg_softc { 588 struct ifnet wg_if; 589 LIST_ENTRY(wg_softc) wg_list; 590 kmutex_t *wg_lock; 591 krwlock_t *wg_rwlock; 592 593 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 594 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 595 596 int wg_npeers; 597 struct pslist_head wg_peers; 598 struct thmap *wg_peers_bypubkey; 599 struct thmap *wg_peers_byname; 600 struct thmap *wg_sessions_byindex; 601 uint16_t wg_listen_port; 602 603 struct wg_worker *wg_worker; 604 lwp_t *wg_worker_lwp; 605 606 struct radix_node_head *wg_rtable_ipv4; 607 struct radix_node_head *wg_rtable_ipv6; 608 609 struct wg_ppsratecheck wg_ppsratecheck; 610 611 struct wg_ops *wg_ops; 612 613 #ifdef WG_RUMPKERNEL 614 struct wg_user *wg_user; 615 #endif 616 }; 617 618 /* [W] 6.1 Preliminaries */ 619 #define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 620 #define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 621 #define WG_REKEY_AFTER_TIME 120 622 #define WG_REJECT_AFTER_TIME 180 623 #define WG_REKEY_ATTEMPT_TIME 90 624 #define WG_REKEY_TIMEOUT 5 625 #define WG_KEEPALIVE_TIMEOUT 10 626 627 #define WG_COOKIE_TIME 120 628 #define WG_RANDVAL_TIME (2 * 60) 629 630 static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 631 static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 632 static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 633 static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 634 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 635 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 636 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 637 638 static struct mbuf * 639 wg_get_mbuf(size_t, size_t); 640 641 static void wg_wakeup_worker(struct wg_worker *, int); 642 643 static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 644 struct mbuf *); 645 static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 646 const uint32_t, const uint8_t [], const struct sockaddr *); 647 static int wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *, 648 struct wg_session *, const struct wg_msg_init *); 649 static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 650 651 static struct wg_peer * 652 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 653 struct psref *); 654 static struct wg_peer * 655 wg_lookup_peer_by_pubkey(struct wg_softc *, 656 const uint8_t [], struct psref *); 657 658 static struct wg_session * 659 wg_lookup_session_by_index(struct wg_softc *, 660 const uint32_t, struct psref *); 661 662 static void wg_update_endpoint_if_necessary(struct wg_peer *, 663 const struct sockaddr *); 664 665 static void wg_schedule_rekey_timer(struct wg_peer *); 666 static void wg_schedule_session_dtor_timer(struct wg_peer *); 667 668 static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 669 static void wg_calculate_keys(struct wg_session *, const bool); 670 671 static void wg_clear_states(struct wg_session *); 672 673 static void wg_get_peer(struct wg_peer *, struct psref *); 674 static void wg_put_peer(struct wg_peer *, struct psref *); 675 676 static int wg_send_so(struct wg_peer *, struct mbuf *); 677 static int wg_send_udp(struct wg_peer *, struct mbuf *); 678 static int wg_output(struct ifnet *, struct mbuf *, 679 const struct sockaddr *, const struct rtentry *); 680 static void wg_input(struct ifnet *, struct mbuf *, const int); 681 static int wg_ioctl(struct ifnet *, u_long, void *); 682 static int wg_bind_port(struct wg_softc *, const uint16_t); 683 static int wg_init(struct ifnet *); 684 static void wg_stop(struct ifnet *, int); 685 686 static void wg_purge_pending_packets(struct wg_peer *); 687 688 static int wg_clone_create(struct if_clone *, int); 689 static int wg_clone_destroy(struct ifnet *); 690 691 struct wg_ops { 692 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 693 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 694 void (*input)(struct ifnet *, struct mbuf *, const int); 695 int (*bind_port)(struct wg_softc *, const uint16_t); 696 }; 697 698 struct wg_ops wg_ops_rumpkernel = { 699 .send_hs_msg = wg_send_so, 700 .send_data_msg = wg_send_udp, 701 .input = wg_input, 702 .bind_port = wg_bind_port, 703 }; 704 705 #ifdef WG_RUMPKERNEL 706 static bool wg_user_mode(struct wg_softc *); 707 static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 708 709 static int wg_send_user(struct wg_peer *, struct mbuf *); 710 static void wg_input_user(struct ifnet *, struct mbuf *, const int); 711 static int wg_bind_port_user(struct wg_softc *, const uint16_t); 712 713 struct wg_ops wg_ops_rumpuser = { 714 .send_hs_msg = wg_send_user, 715 .send_data_msg = wg_send_user, 716 .input = wg_input_user, 717 .bind_port = wg_bind_port_user, 718 }; 719 #endif 720 721 #define WG_PEER_READER_FOREACH(wgp, wg) \ 722 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 723 wgp_peerlist_entry) 724 #define WG_PEER_WRITER_FOREACH(wgp, wg) \ 725 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 726 wgp_peerlist_entry) 727 #define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 728 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 729 #define WG_PEER_WRITER_REMOVE(wgp) \ 730 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 731 732 struct wg_route { 733 struct radix_node wgr_nodes[2]; 734 struct wg_peer *wgr_peer; 735 }; 736 737 static struct radix_node_head * 738 wg_rnh(struct wg_softc *wg, const int family) 739 { 740 741 switch (family) { 742 case AF_INET: 743 return wg->wg_rtable_ipv4; 744 #ifdef INET6 745 case AF_INET6: 746 return wg->wg_rtable_ipv6; 747 #endif 748 default: 749 return NULL; 750 } 751 } 752 753 754 /* 755 * Global variables 756 */ 757 LIST_HEAD(wg_sclist, wg_softc); 758 static struct { 759 struct wg_sclist list; 760 kmutex_t lock; 761 } wg_softcs __cacheline_aligned; 762 763 struct psref_class *wg_psref_class __read_mostly; 764 765 static struct if_clone wg_cloner = 766 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 767 768 769 void wgattach(int); 770 /* ARGSUSED */ 771 void 772 wgattach(int count) 773 { 774 /* 775 * Nothing to do here, initialization is handled by the 776 * module initialization code in wginit() below). 777 */ 778 } 779 780 static void 781 wginit(void) 782 { 783 784 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 785 786 mutex_init(&wg_softcs.lock, MUTEX_DEFAULT, IPL_NONE); 787 LIST_INIT(&wg_softcs.list); 788 if_clone_attach(&wg_cloner); 789 } 790 791 static int 792 wgdetach(void) 793 { 794 int error = 0; 795 796 mutex_enter(&wg_softcs.lock); 797 if (!LIST_EMPTY(&wg_softcs.list)) { 798 mutex_exit(&wg_softcs.lock); 799 error = EBUSY; 800 } 801 802 if (error == 0) { 803 psref_class_destroy(wg_psref_class); 804 805 if_clone_detach(&wg_cloner); 806 } 807 808 return error; 809 } 810 811 static void 812 wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 813 uint8_t hash[WG_HASH_LEN]) 814 { 815 /* [W] 5.4: CONSTRUCTION */ 816 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 817 /* [W] 5.4: IDENTIFIER */ 818 const char *id = "WireGuard v1 zx2c4 Jason@zx2c4.com"; 819 struct blake2s state; 820 821 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 822 signature, strlen(signature)); 823 824 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 825 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 826 827 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 828 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 829 blake2s_update(&state, id, strlen(id)); 830 blake2s_final(&state, hash); 831 832 WG_DUMP_HASH("ckey", ckey); 833 WG_DUMP_HASH("hash", hash); 834 } 835 836 static void 837 wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 838 const size_t inputsize) 839 { 840 struct blake2s state; 841 842 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 843 blake2s_update(&state, hash, WG_HASH_LEN); 844 blake2s_update(&state, input, inputsize); 845 blake2s_final(&state, hash); 846 } 847 848 static void 849 wg_algo_mac(uint8_t out[], const size_t outsize, 850 const uint8_t key[], const size_t keylen, 851 const uint8_t input1[], const size_t input1len, 852 const uint8_t input2[], const size_t input2len) 853 { 854 struct blake2s state; 855 856 blake2s_init(&state, outsize, key, keylen); 857 858 blake2s_update(&state, input1, input1len); 859 if (input2 != NULL) 860 blake2s_update(&state, input2, input2len); 861 blake2s_final(&state, out); 862 } 863 864 static void 865 wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 866 const uint8_t input1[], const size_t input1len, 867 const uint8_t input2[], const size_t input2len) 868 { 869 struct blake2s state; 870 /* [W] 5.4: LABEL-MAC1 */ 871 const char *label = "mac1----"; 872 uint8_t key[WG_HASH_LEN]; 873 874 blake2s_init(&state, sizeof(key), NULL, 0); 875 blake2s_update(&state, label, strlen(label)); 876 blake2s_update(&state, input1, input1len); 877 blake2s_final(&state, key); 878 879 blake2s_init(&state, outsize, key, sizeof(key)); 880 if (input2 != NULL) 881 blake2s_update(&state, input2, input2len); 882 blake2s_final(&state, out); 883 } 884 885 static void 886 wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 887 const uint8_t input1[], const size_t input1len) 888 { 889 struct blake2s state; 890 /* [W] 5.4: LABEL-COOKIE */ 891 const char *label = "cookie--"; 892 893 blake2s_init(&state, outsize, NULL, 0); 894 blake2s_update(&state, label, strlen(label)); 895 blake2s_update(&state, input1, input1len); 896 blake2s_final(&state, out); 897 } 898 899 static void 900 wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 901 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 902 { 903 904 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 905 906 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 907 crypto_scalarmult_base(pubkey, privkey); 908 } 909 910 static void 911 wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 912 const uint8_t privkey[WG_STATIC_KEY_LEN], 913 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 914 { 915 916 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 917 918 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 919 KASSERT(ret == 0); 920 } 921 922 static void 923 wg_algo_hmac(uint8_t out[], const size_t outlen, 924 const uint8_t key[], const size_t keylen, 925 const uint8_t in[], const size_t inlen) 926 { 927 #define IPAD 0x36 928 #define OPAD 0x5c 929 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 930 uint8_t ipad[HMAC_BLOCK_LEN]; 931 uint8_t opad[HMAC_BLOCK_LEN]; 932 int i; 933 struct blake2s state; 934 935 KASSERT(outlen == WG_HASH_LEN); 936 KASSERT(keylen <= HMAC_BLOCK_LEN); 937 938 memcpy(hmackey, key, keylen); 939 940 for (i = 0; i < sizeof(hmackey); i++) { 941 ipad[i] = hmackey[i] ^ IPAD; 942 opad[i] = hmackey[i] ^ OPAD; 943 } 944 945 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 946 blake2s_update(&state, ipad, sizeof(ipad)); 947 blake2s_update(&state, in, inlen); 948 blake2s_final(&state, out); 949 950 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 951 blake2s_update(&state, opad, sizeof(opad)); 952 blake2s_update(&state, out, WG_HASH_LEN); 953 blake2s_final(&state, out); 954 #undef IPAD 955 #undef OPAD 956 } 957 958 static void 959 wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 960 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 961 const uint8_t input[], const size_t inputlen) 962 { 963 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 964 uint8_t one[1]; 965 966 /* 967 * [N] 4.3: "an input_key_material byte sequence with length 968 * either zero bytes, 32 bytes, or DHLEN bytes." 969 */ 970 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 971 972 WG_DUMP_HASH("ckey", ckey); 973 if (input != NULL) 974 WG_DUMP_HASH("input", input); 975 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 976 input, inputlen); 977 WG_DUMP_HASH("tmp1", tmp1); 978 one[0] = 1; 979 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 980 one, sizeof(one)); 981 WG_DUMP_HASH("out1", out1); 982 if (out2 == NULL) 983 return; 984 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 985 tmp2[WG_KDF_OUTPUT_LEN] = 2; 986 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 987 tmp2, sizeof(tmp2)); 988 WG_DUMP_HASH("out2", out2); 989 if (out3 == NULL) 990 return; 991 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 992 tmp2[WG_KDF_OUTPUT_LEN] = 3; 993 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 994 tmp2, sizeof(tmp2)); 995 WG_DUMP_HASH("out3", out3); 996 } 997 998 static void 999 wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1000 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1001 const uint8_t local_key[WG_STATIC_KEY_LEN], 1002 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1003 { 1004 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1005 1006 wg_algo_dh(dhout, local_key, remote_key); 1007 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1008 1009 WG_DUMP_HASH("dhout", dhout); 1010 WG_DUMP_HASH("ckey", ckey); 1011 if (cipher_key != NULL) 1012 WG_DUMP_HASH("cipher_key", cipher_key); 1013 } 1014 1015 static void 1016 wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1017 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1018 const uint8_t auth[], size_t authlen) 1019 { 1020 uint8_t nonce[(32 + 64) / 8] = {0}; 1021 long long unsigned int outsize; 1022 int error __diagused; 1023 1024 le64enc(&nonce[4], counter); 1025 1026 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1027 plainsize, auth, authlen, NULL, nonce, key); 1028 KASSERT(error == 0); 1029 KASSERT(outsize == expected_outsize); 1030 } 1031 1032 static int 1033 wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1034 const uint64_t counter, const uint8_t encrypted[], 1035 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1036 { 1037 uint8_t nonce[(32 + 64) / 8] = {0}; 1038 long long unsigned int outsize; 1039 int error; 1040 1041 le64enc(&nonce[4], counter); 1042 1043 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1044 encrypted, encryptedsize, auth, authlen, nonce, key); 1045 if (error == 0) 1046 KASSERT(outsize == expected_outsize); 1047 return error; 1048 } 1049 1050 static void 1051 wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1052 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1053 const uint8_t auth[], size_t authlen, 1054 const uint8_t nonce[WG_SALT_LEN]) 1055 { 1056 long long unsigned int outsize; 1057 int error __diagused; 1058 1059 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1060 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1061 plain, plainsize, auth, authlen, NULL, nonce, key); 1062 KASSERT(error == 0); 1063 KASSERT(outsize == expected_outsize); 1064 } 1065 1066 static int 1067 wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1068 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1069 const uint8_t auth[], size_t authlen, 1070 const uint8_t nonce[WG_SALT_LEN]) 1071 { 1072 long long unsigned int outsize; 1073 int error; 1074 1075 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1076 encrypted, encryptedsize, auth, authlen, nonce, key); 1077 if (error == 0) 1078 KASSERT(outsize == expected_outsize); 1079 return error; 1080 } 1081 1082 static void 1083 wg_algo_tai64n(wg_timestamp_t timestamp) 1084 { 1085 struct timespec ts; 1086 1087 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1088 getnanotime(&ts); 1089 /* TAI64 label in external TAI64 format */ 1090 be32enc(timestamp, 0x40000000UL + (ts.tv_sec >> 32)); 1091 /* second beginning from 1970 TAI */ 1092 be32enc(timestamp + 4, ts.tv_sec & 0xffffffffU); 1093 /* nanosecond in big-endian format */ 1094 be32enc(timestamp + 8, ts.tv_nsec); 1095 } 1096 1097 /* 1098 * wg_get_stable_session(wgp, psref) 1099 * 1100 * Get a passive reference to the current stable session, or 1101 * return NULL if there is no current stable session. 1102 * 1103 * The pointer is always there but the session is not necessarily 1104 * ESTABLISHED; if it is not ESTABLISHED, return NULL. However, 1105 * the session may transition from ESTABLISHED to DESTROYING while 1106 * holding the passive reference. 1107 */ 1108 static struct wg_session * 1109 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1110 { 1111 int s; 1112 struct wg_session *wgs; 1113 1114 s = pserialize_read_enter(); 1115 wgs = atomic_load_consume(&wgp->wgp_session_stable); 1116 if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED)) 1117 wgs = NULL; 1118 else 1119 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1120 pserialize_read_exit(s); 1121 1122 return wgs; 1123 } 1124 1125 static void 1126 wg_put_session(struct wg_session *wgs, struct psref *psref) 1127 { 1128 1129 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1130 } 1131 1132 static void 1133 wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs) 1134 { 1135 struct wg_peer *wgp = wgs->wgs_peer; 1136 struct wg_session *wgs0 __diagused; 1137 void *garbage; 1138 1139 KASSERT(mutex_owned(wgp->wgp_lock)); 1140 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1141 1142 /* Remove the session from the table. */ 1143 wgs0 = thmap_del(wg->wg_sessions_byindex, 1144 &wgs->wgs_local_index, sizeof(wgs->wgs_local_index)); 1145 KASSERT(wgs0 == wgs); 1146 garbage = thmap_stage_gc(wg->wg_sessions_byindex); 1147 1148 /* Wait for passive references to drain. */ 1149 pserialize_perform(wgp->wgp_psz); 1150 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 1151 1152 /* Free memory, zero state, and transition to UNKNOWN. */ 1153 thmap_gc(wg->wg_sessions_byindex, garbage); 1154 wg_clear_states(wgs); 1155 wgs->wgs_state = WGS_STATE_UNKNOWN; 1156 } 1157 1158 /* 1159 * wg_get_session_index(wg, wgs) 1160 * 1161 * Choose a session index for wgs->wgs_local_index, and store it 1162 * in wg's table of sessions by index. 1163 * 1164 * wgs must be the unstable session of its peer, and must be 1165 * transitioning out of the UNKNOWN state. 1166 */ 1167 static void 1168 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs) 1169 { 1170 struct wg_peer *wgp __diagused = wgs->wgs_peer; 1171 struct wg_session *wgs0; 1172 uint32_t index; 1173 1174 KASSERT(mutex_owned(wgp->wgp_lock)); 1175 KASSERT(wgs == wgp->wgp_session_unstable); 1176 KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN); 1177 1178 do { 1179 /* Pick a uniform random index. */ 1180 index = cprng_strong32(); 1181 1182 /* Try to take it. */ 1183 wgs->wgs_local_index = index; 1184 wgs0 = thmap_put(wg->wg_sessions_byindex, 1185 &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs); 1186 1187 /* If someone else beat us, start over. */ 1188 } while (__predict_false(wgs0 != wgs)); 1189 } 1190 1191 /* 1192 * wg_put_session_index(wg, wgs) 1193 * 1194 * Remove wgs from the table of sessions by index, wait for any 1195 * passive references to drain, and transition the session to the 1196 * UNKNOWN state. 1197 * 1198 * wgs must be the unstable session of its peer, and must not be 1199 * UNKNOWN or ESTABLISHED. 1200 */ 1201 static void 1202 wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs) 1203 { 1204 struct wg_peer *wgp = wgs->wgs_peer; 1205 1206 KASSERT(mutex_owned(wgp->wgp_lock)); 1207 KASSERT(wgs == wgp->wgp_session_unstable); 1208 KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN); 1209 KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED); 1210 1211 wg_destroy_session(wg, wgs); 1212 psref_target_init(&wgs->wgs_psref, wg_psref_class); 1213 } 1214 1215 /* 1216 * Handshake patterns 1217 * 1218 * [W] 5: "These messages use the "IK" pattern from Noise" 1219 * [N] 7.5. Interactive handshake patterns (fundamental) 1220 * "The first character refers to the initiator’s static key:" 1221 * "I = Static key for initiator Immediately transmitted to responder, 1222 * despite reduced or absent identity hiding" 1223 * "The second character refers to the responder’s static key:" 1224 * "K = Static key for responder Known to initiator" 1225 * "IK: 1226 * <- s 1227 * ... 1228 * -> e, es, s, ss 1229 * <- e, ee, se" 1230 * [N] 9.4. Pattern modifiers 1231 * "IKpsk2: 1232 * <- s 1233 * ... 1234 * -> e, es, s, ss 1235 * <- e, ee, se, psk" 1236 */ 1237 static void 1238 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1239 struct wg_session *wgs, struct wg_msg_init *wgmi) 1240 { 1241 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1242 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1243 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1244 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1245 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1246 1247 KASSERT(mutex_owned(wgp->wgp_lock)); 1248 KASSERT(wgs == wgp->wgp_session_unstable); 1249 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1250 1251 wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT); 1252 wgmi->wgmi_sender = wgs->wgs_local_index; 1253 1254 /* [W] 5.4.2: First Message: Initiator to Responder */ 1255 1256 /* Ci := HASH(CONSTRUCTION) */ 1257 /* Hi := HASH(Ci || IDENTIFIER) */ 1258 wg_init_key_and_hash(ckey, hash); 1259 /* Hi := HASH(Hi || Sr^pub) */ 1260 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1261 1262 WG_DUMP_HASH("hash", hash); 1263 1264 /* [N] 2.2: "e" */ 1265 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1266 wg_algo_generate_keypair(pubkey, privkey); 1267 /* Ci := KDF1(Ci, Ei^pub) */ 1268 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1269 /* msg.ephemeral := Ei^pub */ 1270 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1271 /* Hi := HASH(Hi || msg.ephemeral) */ 1272 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1273 1274 WG_DUMP_HASH("ckey", ckey); 1275 WG_DUMP_HASH("hash", hash); 1276 1277 /* [N] 2.2: "es" */ 1278 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1279 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1280 1281 /* [N] 2.2: "s" */ 1282 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1283 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1284 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1285 hash, sizeof(hash)); 1286 /* Hi := HASH(Hi || msg.static) */ 1287 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1288 1289 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1290 1291 /* [N] 2.2: "ss" */ 1292 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1293 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1294 1295 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1296 wg_timestamp_t timestamp; 1297 wg_algo_tai64n(timestamp); 1298 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1299 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1300 /* Hi := HASH(Hi || msg.timestamp) */ 1301 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1302 1303 /* [W] 5.4.4 Cookie MACs */ 1304 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1305 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1306 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1307 /* Need mac1 to decrypt a cookie from a cookie message */ 1308 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1309 sizeof(wgp->wgp_last_sent_mac1)); 1310 wgp->wgp_last_sent_mac1_valid = true; 1311 1312 if (wgp->wgp_latest_cookie_time == 0 || 1313 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1314 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1315 else { 1316 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1317 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1318 (const uint8_t *)wgmi, 1319 offsetof(struct wg_msg_init, wgmi_mac2), 1320 NULL, 0); 1321 } 1322 1323 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1324 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1325 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1326 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1327 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index); 1328 } 1329 1330 static void 1331 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1332 const struct sockaddr *src) 1333 { 1334 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1335 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1336 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1337 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1338 struct wg_peer *wgp; 1339 struct wg_session *wgs; 1340 int error, ret; 1341 struct psref psref_peer; 1342 uint8_t mac1[WG_MAC_LEN]; 1343 1344 WG_TRACE("init msg received"); 1345 1346 wg_algo_mac_mac1(mac1, sizeof(mac1), 1347 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1348 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1349 1350 /* 1351 * [W] 5.3: Denial of Service Mitigation & Cookies 1352 * "the responder, ..., must always reject messages with an invalid 1353 * msg.mac1" 1354 */ 1355 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1356 WG_DLOG("mac1 is invalid\n"); 1357 return; 1358 } 1359 1360 /* 1361 * [W] 5.4.2: First Message: Initiator to Responder 1362 * "When the responder receives this message, it does the same 1363 * operations so that its final state variables are identical, 1364 * replacing the operands of the DH function to produce equivalent 1365 * values." 1366 * Note that the following comments of operations are just copies of 1367 * the initiator's ones. 1368 */ 1369 1370 /* Ci := HASH(CONSTRUCTION) */ 1371 /* Hi := HASH(Ci || IDENTIFIER) */ 1372 wg_init_key_and_hash(ckey, hash); 1373 /* Hi := HASH(Hi || Sr^pub) */ 1374 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1375 1376 /* [N] 2.2: "e" */ 1377 /* Ci := KDF1(Ci, Ei^pub) */ 1378 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1379 sizeof(wgmi->wgmi_ephemeral)); 1380 /* Hi := HASH(Hi || msg.ephemeral) */ 1381 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1382 1383 WG_DUMP_HASH("ckey", ckey); 1384 1385 /* [N] 2.2: "es" */ 1386 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1387 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1388 1389 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1390 1391 /* [N] 2.2: "s" */ 1392 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1393 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1394 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1395 if (error != 0) { 1396 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1397 "wg_algo_aead_dec for secret key failed\n"); 1398 return; 1399 } 1400 /* Hi := HASH(Hi || msg.static) */ 1401 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1402 1403 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1404 if (wgp == NULL) { 1405 WG_DLOG("peer not found\n"); 1406 return; 1407 } 1408 1409 /* 1410 * Lock the peer to serialize access to cookie state. 1411 * 1412 * XXX Can we safely avoid holding the lock across DH? Take it 1413 * just to verify mac2 and then unlock/DH/lock? 1414 */ 1415 mutex_enter(wgp->wgp_lock); 1416 1417 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1418 WG_TRACE("under load"); 1419 /* 1420 * [W] 5.3: Denial of Service Mitigation & Cookies 1421 * "the responder, ..., and when under load may reject messages 1422 * with an invalid msg.mac2. If the responder receives a 1423 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1424 * and is under load, it may respond with a cookie reply 1425 * message" 1426 */ 1427 uint8_t zero[WG_MAC_LEN] = {0}; 1428 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1429 WG_TRACE("sending a cookie message: no cookie included"); 1430 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1431 wgmi->wgmi_mac1, src); 1432 goto out; 1433 } 1434 if (!wgp->wgp_last_sent_cookie_valid) { 1435 WG_TRACE("sending a cookie message: no cookie sent ever"); 1436 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1437 wgmi->wgmi_mac1, src); 1438 goto out; 1439 } 1440 uint8_t mac2[WG_MAC_LEN]; 1441 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1442 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1443 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1444 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1445 WG_DLOG("mac2 is invalid\n"); 1446 goto out; 1447 } 1448 WG_TRACE("under load, but continue to sending"); 1449 } 1450 1451 /* [N] 2.2: "ss" */ 1452 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1453 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1454 1455 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1456 wg_timestamp_t timestamp; 1457 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1458 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1459 hash, sizeof(hash)); 1460 if (error != 0) { 1461 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1462 "wg_algo_aead_dec for timestamp failed\n"); 1463 goto out; 1464 } 1465 /* Hi := HASH(Hi || msg.timestamp) */ 1466 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1467 1468 /* 1469 * [W] 5.1 "The responder keeps track of the greatest timestamp 1470 * received per peer and discards packets containing 1471 * timestamps less than or equal to it." 1472 */ 1473 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1474 sizeof(timestamp)); 1475 if (ret <= 0) { 1476 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1477 "invalid init msg: timestamp is old\n"); 1478 goto out; 1479 } 1480 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1481 1482 /* 1483 * Message is good -- we're committing to handle it now, unless 1484 * we were already initiating a session. 1485 */ 1486 wgs = wgp->wgp_session_unstable; 1487 switch (wgs->wgs_state) { 1488 case WGS_STATE_UNKNOWN: /* new session initiated by peer */ 1489 wg_get_session_index(wg, wgs); 1490 break; 1491 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, drop */ 1492 WG_TRACE("Session already initializing, ignoring the message"); 1493 goto out; 1494 case WGS_STATE_INIT_PASSIVE: /* peer is retrying, start over */ 1495 WG_TRACE("Session already initializing, destroying old states"); 1496 wg_clear_states(wgs); 1497 /* keep session index */ 1498 break; 1499 case WGS_STATE_ESTABLISHED: /* can't happen */ 1500 panic("unstable session can't be established"); 1501 break; 1502 case WGS_STATE_DESTROYING: /* rekey initiated by peer */ 1503 WG_TRACE("Session destroying, but force to clear"); 1504 callout_stop(&wgp->wgp_session_dtor_timer); 1505 wg_clear_states(wgs); 1506 /* keep session index */ 1507 break; 1508 default: 1509 panic("invalid session state: %d", wgs->wgs_state); 1510 } 1511 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1512 1513 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1514 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1515 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1516 sizeof(wgmi->wgmi_ephemeral)); 1517 1518 wg_update_endpoint_if_necessary(wgp, src); 1519 1520 (void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi); 1521 1522 wg_calculate_keys(wgs, false); 1523 wg_clear_states(wgs); 1524 1525 out: 1526 mutex_exit(wgp->wgp_lock); 1527 wg_put_peer(wgp, &psref_peer); 1528 } 1529 1530 static struct socket * 1531 wg_get_so_by_af(struct wg_worker *wgw, const int af) 1532 { 1533 1534 return (af == AF_INET) ? wgw->wgw_so4 : wgw->wgw_so6; 1535 } 1536 1537 static struct socket * 1538 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa) 1539 { 1540 1541 return wg_get_so_by_af(wgp->wgp_sc->wg_worker, wgsa_family(wgsa)); 1542 } 1543 1544 static struct wg_sockaddr * 1545 wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1546 { 1547 struct wg_sockaddr *wgsa; 1548 int s; 1549 1550 s = pserialize_read_enter(); 1551 wgsa = atomic_load_consume(&wgp->wgp_endpoint); 1552 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1553 pserialize_read_exit(s); 1554 1555 return wgsa; 1556 } 1557 1558 static void 1559 wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1560 { 1561 1562 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1563 } 1564 1565 static int 1566 wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1567 { 1568 int error; 1569 struct socket *so; 1570 struct psref psref; 1571 struct wg_sockaddr *wgsa; 1572 1573 wgsa = wg_get_endpoint_sa(wgp, &psref); 1574 so = wg_get_so_by_peer(wgp, wgsa); 1575 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1576 wg_put_sa(wgp, wgsa, &psref); 1577 1578 return error; 1579 } 1580 1581 static int 1582 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1583 { 1584 int error; 1585 struct mbuf *m; 1586 struct wg_msg_init *wgmi; 1587 struct wg_session *wgs; 1588 1589 KASSERT(mutex_owned(wgp->wgp_lock)); 1590 1591 wgs = wgp->wgp_session_unstable; 1592 /* XXX pull dispatch out into wg_task_send_init_message */ 1593 switch (wgs->wgs_state) { 1594 case WGS_STATE_UNKNOWN: /* new session initiated by us */ 1595 wg_get_session_index(wg, wgs); 1596 break; 1597 case WGS_STATE_INIT_ACTIVE: /* we're already initiating, stop */ 1598 WG_TRACE("Session already initializing, skip starting new one"); 1599 return EBUSY; 1600 case WGS_STATE_INIT_PASSIVE: /* peer was trying -- XXX what now? */ 1601 WG_TRACE("Session already initializing, destroying old states"); 1602 wg_clear_states(wgs); 1603 /* keep session index */ 1604 break; 1605 case WGS_STATE_ESTABLISHED: /* can't happen */ 1606 panic("unstable session can't be established"); 1607 break; 1608 case WGS_STATE_DESTROYING: /* rekey initiated by us too early */ 1609 WG_TRACE("Session destroying"); 1610 /* XXX should wait? */ 1611 return EBUSY; 1612 } 1613 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1614 1615 m = m_gethdr(M_WAIT, MT_DATA); 1616 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1617 wgmi = mtod(m, struct wg_msg_init *); 1618 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1619 1620 error = wg->wg_ops->send_hs_msg(wgp, m); 1621 if (error == 0) { 1622 WG_TRACE("init msg sent"); 1623 1624 if (wgp->wgp_handshake_start_time == 0) 1625 wgp->wgp_handshake_start_time = time_uptime; 1626 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1627 MIN(wg_rekey_timeout, INT_MAX/hz) * hz); 1628 } else { 1629 wg_put_session_index(wg, wgs); 1630 } 1631 1632 return error; 1633 } 1634 1635 static void 1636 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1637 struct wg_session *wgs, struct wg_msg_resp *wgmr, 1638 const struct wg_msg_init *wgmi) 1639 { 1640 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1641 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1642 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1643 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1644 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1645 1646 KASSERT(mutex_owned(wgp->wgp_lock)); 1647 KASSERT(wgs == wgp->wgp_session_unstable); 1648 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1649 1650 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1651 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1652 1653 wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP); 1654 wgmr->wgmr_sender = wgs->wgs_local_index; 1655 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1656 1657 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1658 1659 /* [N] 2.2: "e" */ 1660 /* Er^priv, Er^pub := DH-GENERATE() */ 1661 wg_algo_generate_keypair(pubkey, privkey); 1662 /* Cr := KDF1(Cr, Er^pub) */ 1663 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1664 /* msg.ephemeral := Er^pub */ 1665 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1666 /* Hr := HASH(Hr || msg.ephemeral) */ 1667 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1668 1669 WG_DUMP_HASH("ckey", ckey); 1670 WG_DUMP_HASH("hash", hash); 1671 1672 /* [N] 2.2: "ee" */ 1673 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1674 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1675 1676 /* [N] 2.2: "se" */ 1677 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1678 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1679 1680 /* [N] 9.2: "psk" */ 1681 { 1682 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1683 /* Cr, r, k := KDF3(Cr, Q) */ 1684 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1685 sizeof(wgp->wgp_psk)); 1686 /* Hr := HASH(Hr || r) */ 1687 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1688 } 1689 1690 /* msg.empty := AEAD(k, 0, e, Hr) */ 1691 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1692 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1693 /* Hr := HASH(Hr || msg.empty) */ 1694 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1695 1696 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1697 1698 /* [W] 5.4.4: Cookie MACs */ 1699 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1700 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1701 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1702 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1703 /* Need mac1 to decrypt a cookie from a cookie message */ 1704 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1705 sizeof(wgp->wgp_last_sent_mac1)); 1706 wgp->wgp_last_sent_mac1_valid = true; 1707 1708 if (wgp->wgp_latest_cookie_time == 0 || 1709 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1710 /* msg.mac2 := 0^16 */ 1711 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1712 else { 1713 /* msg.mac2 := MAC(Lm, msg_b) */ 1714 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1715 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1716 (const uint8_t *)wgmr, 1717 offsetof(struct wg_msg_resp, wgmr_mac2), 1718 NULL, 0); 1719 } 1720 1721 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1722 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1723 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1724 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1725 wgs->wgs_remote_index = wgmi->wgmi_sender; 1726 WG_DLOG("sender=%x\n", wgs->wgs_local_index); 1727 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1728 } 1729 1730 static void 1731 wg_swap_sessions(struct wg_peer *wgp) 1732 { 1733 struct wg_session *wgs, *wgs_prev; 1734 1735 KASSERT(mutex_owned(wgp->wgp_lock)); 1736 1737 wgs = wgp->wgp_session_unstable; 1738 KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED); 1739 1740 wgs_prev = wgp->wgp_session_stable; 1741 KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED || 1742 wgs_prev->wgs_state == WGS_STATE_UNKNOWN); 1743 atomic_store_release(&wgp->wgp_session_stable, wgs); 1744 wgp->wgp_session_unstable = wgs_prev; 1745 } 1746 1747 static void 1748 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1749 const struct sockaddr *src) 1750 { 1751 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1752 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1753 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1754 struct wg_peer *wgp; 1755 struct wg_session *wgs; 1756 struct psref psref; 1757 int error; 1758 uint8_t mac1[WG_MAC_LEN]; 1759 struct wg_session *wgs_prev; 1760 1761 wg_algo_mac_mac1(mac1, sizeof(mac1), 1762 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1763 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1764 1765 /* 1766 * [W] 5.3: Denial of Service Mitigation & Cookies 1767 * "the responder, ..., must always reject messages with an invalid 1768 * msg.mac1" 1769 */ 1770 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1771 WG_DLOG("mac1 is invalid\n"); 1772 return; 1773 } 1774 1775 WG_TRACE("resp msg received"); 1776 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1777 if (wgs == NULL) { 1778 WG_TRACE("No session found"); 1779 return; 1780 } 1781 1782 wgp = wgs->wgs_peer; 1783 1784 mutex_enter(wgp->wgp_lock); 1785 1786 /* If we weren't waiting for a handshake response, drop it. */ 1787 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) { 1788 WG_TRACE("peer sent spurious handshake response, ignoring"); 1789 goto out; 1790 } 1791 1792 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1793 WG_TRACE("under load"); 1794 /* 1795 * [W] 5.3: Denial of Service Mitigation & Cookies 1796 * "the responder, ..., and when under load may reject messages 1797 * with an invalid msg.mac2. If the responder receives a 1798 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1799 * and is under load, it may respond with a cookie reply 1800 * message" 1801 */ 1802 uint8_t zero[WG_MAC_LEN] = {0}; 1803 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1804 WG_TRACE("sending a cookie message: no cookie included"); 1805 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1806 wgmr->wgmr_mac1, src); 1807 goto out; 1808 } 1809 if (!wgp->wgp_last_sent_cookie_valid) { 1810 WG_TRACE("sending a cookie message: no cookie sent ever"); 1811 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1812 wgmr->wgmr_mac1, src); 1813 goto out; 1814 } 1815 uint8_t mac2[WG_MAC_LEN]; 1816 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1817 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1818 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1819 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1820 WG_DLOG("mac2 is invalid\n"); 1821 goto out; 1822 } 1823 WG_TRACE("under load, but continue to sending"); 1824 } 1825 1826 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1827 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1828 1829 /* 1830 * [W] 5.4.3 Second Message: Responder to Initiator 1831 * "When the initiator receives this message, it does the same 1832 * operations so that its final state variables are identical, 1833 * replacing the operands of the DH function to produce equivalent 1834 * values." 1835 * Note that the following comments of operations are just copies of 1836 * the initiator's ones. 1837 */ 1838 1839 /* [N] 2.2: "e" */ 1840 /* Cr := KDF1(Cr, Er^pub) */ 1841 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1842 sizeof(wgmr->wgmr_ephemeral)); 1843 /* Hr := HASH(Hr || msg.ephemeral) */ 1844 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1845 1846 WG_DUMP_HASH("ckey", ckey); 1847 WG_DUMP_HASH("hash", hash); 1848 1849 /* [N] 2.2: "ee" */ 1850 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1851 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1852 wgmr->wgmr_ephemeral); 1853 1854 /* [N] 2.2: "se" */ 1855 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1856 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1857 1858 /* [N] 9.2: "psk" */ 1859 { 1860 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1861 /* Cr, r, k := KDF3(Cr, Q) */ 1862 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1863 sizeof(wgp->wgp_psk)); 1864 /* Hr := HASH(Hr || r) */ 1865 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1866 } 1867 1868 { 1869 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 1870 /* msg.empty := AEAD(k, 0, e, Hr) */ 1871 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 1872 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 1873 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1874 if (error != 0) { 1875 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1876 "wg_algo_aead_dec for empty message failed\n"); 1877 goto out; 1878 } 1879 /* Hr := HASH(Hr || msg.empty) */ 1880 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1881 } 1882 1883 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 1884 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 1885 wgs->wgs_remote_index = wgmr->wgmr_sender; 1886 WG_DLOG("receiver=%x\n", wgs->wgs_remote_index); 1887 1888 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1889 wgs->wgs_state = WGS_STATE_ESTABLISHED; 1890 wgs->wgs_time_established = time_uptime; 1891 wgs->wgs_time_last_data_sent = 0; 1892 wgs->wgs_is_initiator = true; 1893 wg_calculate_keys(wgs, true); 1894 wg_clear_states(wgs); 1895 WG_TRACE("WGS_STATE_ESTABLISHED"); 1896 1897 callout_stop(&wgp->wgp_handshake_timeout_timer); 1898 1899 wg_swap_sessions(wgp); 1900 KASSERT(wgs == wgp->wgp_session_stable); 1901 wgs_prev = wgp->wgp_session_unstable; 1902 getnanotime(&wgp->wgp_last_handshake_time); 1903 wgp->wgp_handshake_start_time = 0; 1904 wgp->wgp_last_sent_mac1_valid = false; 1905 wgp->wgp_last_sent_cookie_valid = false; 1906 1907 wg_schedule_rekey_timer(wgp); 1908 1909 wg_update_endpoint_if_necessary(wgp, src); 1910 1911 /* 1912 * Send something immediately (same as the official implementation) 1913 * XXX if there are pending data packets, we don't need to send 1914 * a keepalive message. 1915 */ 1916 wg_send_keepalive_msg(wgp, wgs); 1917 1918 /* Anyway run a softint to flush pending packets */ 1919 kpreempt_disable(); 1920 softint_schedule(wgp->wgp_si); 1921 kpreempt_enable(); 1922 WG_TRACE("softint scheduled"); 1923 1924 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 1925 /* Wait for wg_get_stable_session to drain. */ 1926 pserialize_perform(wgp->wgp_psz); 1927 1928 /* Transition ESTABLISHED->DESTROYING. */ 1929 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 1930 1931 /* We can't destroy the old session immediately */ 1932 wg_schedule_session_dtor_timer(wgp); 1933 } else { 1934 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 1935 "state=%d", wgs_prev->wgs_state); 1936 } 1937 1938 out: 1939 mutex_exit(wgp->wgp_lock); 1940 wg_put_session(wgs, &psref); 1941 } 1942 1943 static int 1944 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1945 struct wg_session *wgs, const struct wg_msg_init *wgmi) 1946 { 1947 int error; 1948 struct mbuf *m; 1949 struct wg_msg_resp *wgmr; 1950 1951 KASSERT(mutex_owned(wgp->wgp_lock)); 1952 KASSERT(wgs == wgp->wgp_session_unstable); 1953 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1954 1955 m = m_gethdr(M_WAIT, MT_DATA); 1956 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 1957 wgmr = mtod(m, struct wg_msg_resp *); 1958 wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi); 1959 1960 error = wg->wg_ops->send_hs_msg(wgp, m); 1961 if (error == 0) 1962 WG_TRACE("resp msg sent"); 1963 return error; 1964 } 1965 1966 static struct wg_peer * 1967 wg_lookup_peer_by_pubkey(struct wg_softc *wg, 1968 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 1969 { 1970 struct wg_peer *wgp; 1971 1972 int s = pserialize_read_enter(); 1973 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN); 1974 if (wgp != NULL) 1975 wg_get_peer(wgp, psref); 1976 pserialize_read_exit(s); 1977 1978 return wgp; 1979 } 1980 1981 static void 1982 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 1983 struct wg_msg_cookie *wgmc, const uint32_t sender, 1984 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 1985 { 1986 uint8_t cookie[WG_COOKIE_LEN]; 1987 uint8_t key[WG_HASH_LEN]; 1988 uint8_t addr[sizeof(struct in6_addr)]; 1989 size_t addrlen; 1990 uint16_t uh_sport; /* be */ 1991 1992 KASSERT(mutex_owned(wgp->wgp_lock)); 1993 1994 wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE); 1995 wgmc->wgmc_receiver = sender; 1996 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 1997 1998 /* 1999 * [W] 5.4.7: Under Load: Cookie Reply Message 2000 * "The secret variable, Rm, changes every two minutes to a 2001 * random value" 2002 */ 2003 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 2004 wgp->wgp_randval = cprng_strong32(); 2005 wgp->wgp_last_genrandval_time = time_uptime; 2006 } 2007 2008 switch (src->sa_family) { 2009 case AF_INET: { 2010 const struct sockaddr_in *sin = satocsin(src); 2011 addrlen = sizeof(sin->sin_addr); 2012 memcpy(addr, &sin->sin_addr, addrlen); 2013 uh_sport = sin->sin_port; 2014 break; 2015 } 2016 #ifdef INET6 2017 case AF_INET6: { 2018 const struct sockaddr_in6 *sin6 = satocsin6(src); 2019 addrlen = sizeof(sin6->sin6_addr); 2020 memcpy(addr, &sin6->sin6_addr, addrlen); 2021 uh_sport = sin6->sin6_port; 2022 break; 2023 } 2024 #endif 2025 default: 2026 panic("invalid af=%d", src->sa_family); 2027 } 2028 2029 wg_algo_mac(cookie, sizeof(cookie), 2030 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2031 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2032 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2033 sizeof(wg->wg_pubkey)); 2034 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2035 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2036 2037 /* Need to store to calculate mac2 */ 2038 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2039 wgp->wgp_last_sent_cookie_valid = true; 2040 } 2041 2042 static int 2043 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2044 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2045 const struct sockaddr *src) 2046 { 2047 int error; 2048 struct mbuf *m; 2049 struct wg_msg_cookie *wgmc; 2050 2051 KASSERT(mutex_owned(wgp->wgp_lock)); 2052 2053 m = m_gethdr(M_WAIT, MT_DATA); 2054 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2055 wgmc = mtod(m, struct wg_msg_cookie *); 2056 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2057 2058 error = wg->wg_ops->send_hs_msg(wgp, m); 2059 if (error == 0) 2060 WG_TRACE("cookie msg sent"); 2061 return error; 2062 } 2063 2064 static bool 2065 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2066 { 2067 #ifdef WG_DEBUG_PARAMS 2068 if (wg_force_underload) 2069 return true; 2070 #endif 2071 2072 /* 2073 * XXX we don't have a means of a load estimation. The purpose of 2074 * the mechanism is a DoS mitigation, so we consider frequent handshake 2075 * messages as (a kind of) load; if a message of the same type comes 2076 * to a peer within 1 second, we consider we are under load. 2077 */ 2078 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2079 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2080 return (time_uptime - last) == 0; 2081 } 2082 2083 static void 2084 wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2085 { 2086 2087 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2088 2089 /* 2090 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2091 */ 2092 if (initiator) { 2093 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2094 wgs->wgs_chaining_key, NULL, 0); 2095 } else { 2096 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2097 wgs->wgs_chaining_key, NULL, 0); 2098 } 2099 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2100 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2101 } 2102 2103 static uint64_t 2104 wg_session_get_send_counter(struct wg_session *wgs) 2105 { 2106 #ifdef __HAVE_ATOMIC64_LOADSTORE 2107 return atomic_load_relaxed(&wgs->wgs_send_counter); 2108 #else 2109 uint64_t send_counter; 2110 2111 mutex_enter(&wgs->wgs_send_counter_lock); 2112 send_counter = wgs->wgs_send_counter; 2113 mutex_exit(&wgs->wgs_send_counter_lock); 2114 2115 return send_counter; 2116 #endif 2117 } 2118 2119 static uint64_t 2120 wg_session_inc_send_counter(struct wg_session *wgs) 2121 { 2122 #ifdef __HAVE_ATOMIC64_LOADSTORE 2123 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2124 #else 2125 uint64_t send_counter; 2126 2127 mutex_enter(&wgs->wgs_send_counter_lock); 2128 send_counter = wgs->wgs_send_counter++; 2129 mutex_exit(&wgs->wgs_send_counter_lock); 2130 2131 return send_counter; 2132 #endif 2133 } 2134 2135 static void 2136 wg_clear_states(struct wg_session *wgs) 2137 { 2138 2139 KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock)); 2140 2141 wgs->wgs_send_counter = 0; 2142 sliwin_reset(&wgs->wgs_recvwin->window); 2143 2144 #define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2145 wgs_clear(handshake_hash); 2146 wgs_clear(chaining_key); 2147 wgs_clear(ephemeral_key_pub); 2148 wgs_clear(ephemeral_key_priv); 2149 wgs_clear(ephemeral_key_peer); 2150 #undef wgs_clear 2151 } 2152 2153 static struct wg_session * 2154 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2155 struct psref *psref) 2156 { 2157 struct wg_session *wgs; 2158 2159 int s = pserialize_read_enter(); 2160 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index); 2161 if (wgs != NULL) { 2162 KASSERT(atomic_load_relaxed(&wgs->wgs_state) != 2163 WGS_STATE_UNKNOWN); 2164 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2165 } 2166 pserialize_read_exit(s); 2167 2168 return wgs; 2169 } 2170 2171 static void 2172 wg_schedule_rekey_timer(struct wg_peer *wgp) 2173 { 2174 int timeout = MIN(wg_rekey_after_time, INT_MAX/hz); 2175 2176 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2177 } 2178 2179 static void 2180 wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2181 { 2182 struct mbuf *m; 2183 2184 /* 2185 * [W] 6.5 Passive Keepalive 2186 * "A keepalive message is simply a transport data message with 2187 * a zero-length encapsulated encrypted inner-packet." 2188 */ 2189 m = m_gethdr(M_WAIT, MT_DATA); 2190 wg_send_data_msg(wgp, wgs, m); 2191 } 2192 2193 static bool 2194 wg_need_to_send_init_message(struct wg_session *wgs) 2195 { 2196 /* 2197 * [W] 6.2 Transport Message Limits 2198 * "if a peer is the initiator of a current secure session, 2199 * WireGuard will send a handshake initiation message to begin 2200 * a new secure session ... if after receiving a transport data 2201 * message, the current secure session is (REJECT-AFTER-TIME − 2202 * KEEPALIVE-TIMEOUT − REKEY-TIMEOUT) seconds old and it has 2203 * not yet acted upon this event." 2204 */ 2205 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2206 (time_uptime - wgs->wgs_time_established) >= 2207 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2208 } 2209 2210 static void 2211 wg_schedule_peer_task(struct wg_peer *wgp, int task) 2212 { 2213 2214 atomic_or_uint(&wgp->wgp_tasks, task); 2215 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2216 wg_wakeup_worker(wgp->wgp_sc->wg_worker, WG_WAKEUP_REASON_PEER); 2217 } 2218 2219 static void 2220 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2221 { 2222 struct wg_sockaddr *wgsa_prev; 2223 2224 WG_TRACE("Changing endpoint"); 2225 2226 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2227 wgsa_prev = wgp->wgp_endpoint; 2228 atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0); 2229 wgp->wgp_endpoint0 = wgsa_prev; 2230 atomic_store_release(&wgp->wgp_endpoint_available, true); 2231 2232 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2233 } 2234 2235 static bool 2236 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2237 { 2238 uint16_t packet_len; 2239 const struct ip *ip; 2240 2241 if (__predict_false(decrypted_len < sizeof(struct ip))) 2242 return false; 2243 2244 ip = (const struct ip *)packet; 2245 if (ip->ip_v == 4) 2246 *af = AF_INET; 2247 else if (ip->ip_v == 6) 2248 *af = AF_INET6; 2249 else 2250 return false; 2251 2252 WG_DLOG("af=%d\n", *af); 2253 2254 if (*af == AF_INET) { 2255 packet_len = ntohs(ip->ip_len); 2256 } else { 2257 const struct ip6_hdr *ip6; 2258 2259 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr))) 2260 return false; 2261 2262 ip6 = (const struct ip6_hdr *)packet; 2263 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen); 2264 } 2265 2266 WG_DLOG("packet_len=%u\n", packet_len); 2267 if (packet_len > decrypted_len) 2268 return false; 2269 2270 return true; 2271 } 2272 2273 static bool 2274 wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2275 int af, char *packet) 2276 { 2277 struct sockaddr_storage ss; 2278 struct sockaddr *sa; 2279 struct psref psref; 2280 struct wg_peer *wgp; 2281 bool ok; 2282 2283 /* 2284 * II CRYPTOKEY ROUTING 2285 * "it will only accept it if its source IP resolves in the 2286 * table to the public key used in the secure session for 2287 * decrypting it." 2288 */ 2289 2290 if (af == AF_INET) { 2291 const struct ip *ip = (const struct ip *)packet; 2292 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2293 sockaddr_in_init(sin, &ip->ip_src, 0); 2294 sa = sintosa(sin); 2295 #ifdef INET6 2296 } else { 2297 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2298 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2299 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2300 sa = sin6tosa(sin6); 2301 #endif 2302 } 2303 2304 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2305 ok = (wgp == wgp_expected); 2306 if (wgp != NULL) 2307 wg_put_peer(wgp, &psref); 2308 2309 return ok; 2310 } 2311 2312 static void 2313 wg_session_dtor_timer(void *arg) 2314 { 2315 struct wg_peer *wgp = arg; 2316 2317 WG_TRACE("enter"); 2318 2319 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2320 } 2321 2322 static void 2323 wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2324 { 2325 2326 /* 1 second grace period */ 2327 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2328 } 2329 2330 static bool 2331 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2332 { 2333 if (sa1->sa_family != sa2->sa_family) 2334 return false; 2335 2336 switch (sa1->sa_family) { 2337 case AF_INET: 2338 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2339 case AF_INET6: 2340 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2341 default: 2342 return true; 2343 } 2344 } 2345 2346 static void 2347 wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2348 const struct sockaddr *src) 2349 { 2350 struct wg_sockaddr *wgsa; 2351 struct psref psref; 2352 2353 wgsa = wg_get_endpoint_sa(wgp, &psref); 2354 2355 #ifdef WG_DEBUG_LOG 2356 char oldaddr[128], newaddr[128]; 2357 sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr)); 2358 sockaddr_format(src, newaddr, sizeof(newaddr)); 2359 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2360 #endif 2361 2362 /* 2363 * III: "Since the packet has authenticated correctly, the source IP of 2364 * the outer UDP/IP packet is used to update the endpoint for peer..." 2365 */ 2366 if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 || 2367 !sockaddr_port_match(src, wgsatosa(wgsa)))) { 2368 /* XXX We can't change the endpoint twice in a short period */ 2369 if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) { 2370 wg_change_endpoint(wgp, src); 2371 } 2372 } 2373 2374 wg_put_sa(wgp, wgsa, &psref); 2375 } 2376 2377 static void 2378 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2379 const struct sockaddr *src) 2380 { 2381 struct wg_msg_data *wgmd; 2382 char *encrypted_buf = NULL, *decrypted_buf; 2383 size_t encrypted_len, decrypted_len; 2384 struct wg_session *wgs; 2385 struct wg_peer *wgp; 2386 int state; 2387 size_t mlen; 2388 struct psref psref; 2389 int error, af; 2390 bool success, free_encrypted_buf = false, ok; 2391 struct mbuf *n; 2392 2393 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2394 wgmd = mtod(m, struct wg_msg_data *); 2395 2396 KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA)); 2397 WG_TRACE("data"); 2398 2399 /* Find the putative session, or drop. */ 2400 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2401 if (wgs == NULL) { 2402 WG_TRACE("No session found"); 2403 m_freem(m); 2404 return; 2405 } 2406 2407 /* 2408 * We are only ready to handle data when in INIT_PASSIVE, 2409 * ESTABLISHED, or DESTROYING. All transitions out of that 2410 * state dissociate the session index and drain psrefs. 2411 */ 2412 state = atomic_load_relaxed(&wgs->wgs_state); 2413 switch (state) { 2414 case WGS_STATE_UNKNOWN: 2415 panic("wg session %p in unknown state has session index %u", 2416 wgs, wgmd->wgmd_receiver); 2417 case WGS_STATE_INIT_ACTIVE: 2418 WG_TRACE("not yet ready for data"); 2419 goto out; 2420 case WGS_STATE_INIT_PASSIVE: 2421 case WGS_STATE_ESTABLISHED: 2422 case WGS_STATE_DESTROYING: 2423 break; 2424 } 2425 2426 /* 2427 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and 2428 * to update the endpoint if authentication succeeds. 2429 */ 2430 wgp = wgs->wgs_peer; 2431 2432 /* 2433 * Reject outrageously wrong sequence numbers before doing any 2434 * crypto work or taking any locks. 2435 */ 2436 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2437 le64toh(wgmd->wgmd_counter)); 2438 if (error) { 2439 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2440 "out-of-window packet: %"PRIu64"\n", 2441 le64toh(wgmd->wgmd_counter)); 2442 goto out; 2443 } 2444 2445 /* Ensure the payload and authenticator are contiguous. */ 2446 mlen = m_length(m); 2447 encrypted_len = mlen - sizeof(*wgmd); 2448 if (encrypted_len < WG_AUTHTAG_LEN) { 2449 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2450 goto out; 2451 } 2452 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2453 if (success) { 2454 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2455 } else { 2456 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2457 if (encrypted_buf == NULL) { 2458 WG_DLOG("failed to allocate encrypted_buf\n"); 2459 goto out; 2460 } 2461 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2462 free_encrypted_buf = true; 2463 } 2464 /* m_ensure_contig may change m regardless of its result */ 2465 KASSERT(m->m_len >= sizeof(*wgmd)); 2466 wgmd = mtod(m, struct wg_msg_data *); 2467 2468 /* 2469 * Get a buffer for the plaintext. Add WG_AUTHTAG_LEN to avoid 2470 * a zero-length buffer (XXX). Drop if plaintext is longer 2471 * than MCLBYTES (XXX). 2472 */ 2473 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2474 if (decrypted_len > MCLBYTES) { 2475 /* FIXME handle larger data than MCLBYTES */ 2476 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2477 goto out; 2478 } 2479 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2480 if (n == NULL) { 2481 WG_DLOG("wg_get_mbuf failed\n"); 2482 goto out; 2483 } 2484 decrypted_buf = mtod(n, char *); 2485 2486 /* Decrypt and verify the packet. */ 2487 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2488 error = wg_algo_aead_dec(decrypted_buf, 2489 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2490 wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf, 2491 encrypted_len, NULL, 0); 2492 if (error != 0) { 2493 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2494 "failed to wg_algo_aead_dec\n"); 2495 m_freem(n); 2496 goto out; 2497 } 2498 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2499 2500 /* Packet is genuine. Reject it if a replay or just too old. */ 2501 mutex_enter(&wgs->wgs_recvwin->lock); 2502 error = sliwin_update(&wgs->wgs_recvwin->window, 2503 le64toh(wgmd->wgmd_counter)); 2504 mutex_exit(&wgs->wgs_recvwin->lock); 2505 if (error) { 2506 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2507 "replay or out-of-window packet: %"PRIu64"\n", 2508 le64toh(wgmd->wgmd_counter)); 2509 m_freem(n); 2510 goto out; 2511 } 2512 2513 /* We're done with m now; free it and chuck the pointers. */ 2514 m_freem(m); 2515 m = NULL; 2516 wgmd = NULL; 2517 2518 /* 2519 * Validate the encapsulated packet header and get the address 2520 * family, or drop. 2521 */ 2522 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2523 if (!ok) { 2524 m_freem(n); 2525 goto out; 2526 } 2527 2528 /* 2529 * The packet is genuine. Update the peer's endpoint if the 2530 * source address changed. 2531 * 2532 * XXX How to prevent DoS by replaying genuine packets from the 2533 * wrong source address? 2534 */ 2535 wg_update_endpoint_if_necessary(wgp, src); 2536 2537 /* Submit it into our network stack if routable. */ 2538 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2539 if (ok) { 2540 wg->wg_ops->input(&wg->wg_if, n, af); 2541 } else { 2542 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2543 "invalid source address\n"); 2544 m_freem(n); 2545 /* 2546 * The inner address is invalid however the session is valid 2547 * so continue the session processing below. 2548 */ 2549 } 2550 n = NULL; 2551 2552 /* Update the state machine if necessary. */ 2553 if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) { 2554 /* 2555 * We were waiting for the initiator to send their 2556 * first data transport message, and that has happened. 2557 * Schedule a task to establish this session. 2558 */ 2559 wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION); 2560 } else { 2561 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2562 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2563 } 2564 /* 2565 * [W] 6.5 Passive Keepalive 2566 * "If a peer has received a validly-authenticated transport 2567 * data message (section 5.4.6), but does not have any packets 2568 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2569 * a keepalive message." 2570 */ 2571 WG_DLOG("time_uptime=%ju wgs_time_last_data_sent=%ju\n", 2572 (uintmax_t)time_uptime, 2573 (uintmax_t)wgs->wgs_time_last_data_sent); 2574 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2575 wg_keepalive_timeout) { 2576 WG_TRACE("Schedule sending keepalive message"); 2577 /* 2578 * We can't send a keepalive message here to avoid 2579 * a deadlock; we already hold the solock of a socket 2580 * that is used to send the message. 2581 */ 2582 wg_schedule_peer_task(wgp, 2583 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2584 } 2585 } 2586 out: 2587 wg_put_session(wgs, &psref); 2588 if (m != NULL) 2589 m_freem(m); 2590 if (free_encrypted_buf) 2591 kmem_intr_free(encrypted_buf, encrypted_len); 2592 } 2593 2594 static void 2595 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2596 { 2597 struct wg_session *wgs; 2598 struct wg_peer *wgp; 2599 struct psref psref; 2600 int error; 2601 uint8_t key[WG_HASH_LEN]; 2602 uint8_t cookie[WG_COOKIE_LEN]; 2603 2604 WG_TRACE("cookie msg received"); 2605 2606 /* Find the putative session. */ 2607 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2608 if (wgs == NULL) { 2609 WG_TRACE("No session found"); 2610 return; 2611 } 2612 2613 /* Lock the peer so we can update the cookie state. */ 2614 wgp = wgs->wgs_peer; 2615 mutex_enter(wgp->wgp_lock); 2616 2617 if (!wgp->wgp_last_sent_mac1_valid) { 2618 WG_TRACE("No valid mac1 sent (or expired)"); 2619 goto out; 2620 } 2621 2622 /* Decrypt the cookie and store it for later handshake retry. */ 2623 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2624 sizeof(wgp->wgp_pubkey)); 2625 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2626 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2627 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2628 wgmc->wgmc_salt); 2629 if (error != 0) { 2630 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2631 "wg_algo_aead_dec for cookie failed: error=%d\n", error); 2632 goto out; 2633 } 2634 /* 2635 * [W] 6.6: Interaction with Cookie Reply System 2636 * "it should simply store the decrypted cookie value from the cookie 2637 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2638 * timer for retrying a handshake initiation message." 2639 */ 2640 wgp->wgp_latest_cookie_time = time_uptime; 2641 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2642 out: 2643 mutex_exit(wgp->wgp_lock); 2644 wg_put_session(wgs, &psref); 2645 } 2646 2647 static struct mbuf * 2648 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2649 { 2650 struct wg_msg wgm; 2651 size_t mbuflen; 2652 size_t msglen; 2653 2654 /* 2655 * Get the mbuf chain length. It is already guaranteed, by 2656 * wg_overudp_cb, to be large enough for a struct wg_msg. 2657 */ 2658 mbuflen = m_length(m); 2659 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2660 2661 /* 2662 * Copy the message header (32-bit message type) out -- we'll 2663 * worry about contiguity and alignment later. 2664 */ 2665 m_copydata(m, 0, sizeof(wgm), &wgm); 2666 switch (le32toh(wgm.wgm_type)) { 2667 case WG_MSG_TYPE_INIT: 2668 msglen = sizeof(struct wg_msg_init); 2669 break; 2670 case WG_MSG_TYPE_RESP: 2671 msglen = sizeof(struct wg_msg_resp); 2672 break; 2673 case WG_MSG_TYPE_COOKIE: 2674 msglen = sizeof(struct wg_msg_cookie); 2675 break; 2676 case WG_MSG_TYPE_DATA: 2677 msglen = sizeof(struct wg_msg_data); 2678 break; 2679 default: 2680 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2681 "Unexpected msg type: %u\n", le32toh(wgm.wgm_type)); 2682 goto error; 2683 } 2684 2685 /* Verify the mbuf chain is long enough for this type of message. */ 2686 if (__predict_false(mbuflen < msglen)) { 2687 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2688 le32toh(wgm.wgm_type)); 2689 goto error; 2690 } 2691 2692 /* Make the message header contiguous if necessary. */ 2693 if (__predict_false(m->m_len < msglen)) { 2694 m = m_pullup(m, msglen); 2695 if (m == NULL) 2696 return NULL; 2697 } 2698 2699 return m; 2700 2701 error: 2702 m_freem(m); 2703 return NULL; 2704 } 2705 2706 static void 2707 wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2708 const struct sockaddr *src) 2709 { 2710 struct wg_msg *wgm; 2711 2712 m = wg_validate_msg_header(wg, m); 2713 if (__predict_false(m == NULL)) 2714 return; 2715 2716 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2717 wgm = mtod(m, struct wg_msg *); 2718 switch (le32toh(wgm->wgm_type)) { 2719 case WG_MSG_TYPE_INIT: 2720 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2721 break; 2722 case WG_MSG_TYPE_RESP: 2723 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2724 break; 2725 case WG_MSG_TYPE_COOKIE: 2726 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2727 break; 2728 case WG_MSG_TYPE_DATA: 2729 wg_handle_msg_data(wg, m, src); 2730 /* wg_handle_msg_data frees m for us */ 2731 return; 2732 default: 2733 panic("invalid message type: %d", le32toh(wgm->wgm_type)); 2734 } 2735 2736 m_freem(m); 2737 } 2738 2739 static void 2740 wg_receive_packets(struct wg_softc *wg, const int af) 2741 { 2742 2743 for (;;) { 2744 int error, flags; 2745 struct socket *so; 2746 struct mbuf *m = NULL; 2747 struct uio dummy_uio; 2748 struct mbuf *paddr = NULL; 2749 struct sockaddr *src; 2750 2751 so = wg_get_so_by_af(wg->wg_worker, af); 2752 flags = MSG_DONTWAIT; 2753 dummy_uio.uio_resid = 1000000000; 2754 2755 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2756 &flags); 2757 if (error || m == NULL) { 2758 //if (error == EWOULDBLOCK) 2759 return; 2760 } 2761 2762 KASSERT(paddr != NULL); 2763 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2764 src = mtod(paddr, struct sockaddr *); 2765 2766 wg_handle_packet(wg, m, src); 2767 } 2768 } 2769 2770 static void 2771 wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2772 { 2773 2774 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2775 } 2776 2777 static void 2778 wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2779 { 2780 2781 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 2782 } 2783 2784 static void 2785 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 2786 { 2787 struct wg_session *wgs; 2788 2789 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 2790 2791 KASSERT(mutex_owned(wgp->wgp_lock)); 2792 2793 if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) { 2794 WGLOG(LOG_DEBUG, "No endpoint available\n"); 2795 /* XXX should do something? */ 2796 return; 2797 } 2798 2799 wgs = wgp->wgp_session_stable; 2800 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 2801 /* XXX What if the unstable session is already INIT_ACTIVE? */ 2802 wg_send_handshake_msg_init(wg, wgp); 2803 } else { 2804 /* rekey */ 2805 wgs = wgp->wgp_session_unstable; 2806 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2807 wg_send_handshake_msg_init(wg, wgp); 2808 } 2809 } 2810 2811 static void 2812 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp) 2813 { 2814 struct wg_session *wgs; 2815 2816 WG_TRACE("WGP_TASK_RETRY_HANDSHAKE"); 2817 2818 KASSERT(mutex_owned(wgp->wgp_lock)); 2819 KASSERT(wgp->wgp_handshake_start_time != 0); 2820 2821 wgs = wgp->wgp_session_unstable; 2822 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2823 return; 2824 2825 /* 2826 * XXX no real need to assign a new index here, but we do need 2827 * to transition to UNKNOWN temporarily 2828 */ 2829 wg_put_session_index(wg, wgs); 2830 2831 /* [W] 6.4 Handshake Initiation Retransmission */ 2832 if ((time_uptime - wgp->wgp_handshake_start_time) > 2833 wg_rekey_attempt_time) { 2834 /* Give up handshaking */ 2835 wgp->wgp_handshake_start_time = 0; 2836 WG_TRACE("give up"); 2837 2838 /* 2839 * If a new data packet comes, handshaking will be retried 2840 * and a new session would be established at that time, 2841 * however we don't want to send pending packets then. 2842 */ 2843 wg_purge_pending_packets(wgp); 2844 return; 2845 } 2846 2847 wg_task_send_init_message(wg, wgp); 2848 } 2849 2850 static void 2851 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp) 2852 { 2853 struct wg_session *wgs, *wgs_prev; 2854 2855 KASSERT(mutex_owned(wgp->wgp_lock)); 2856 2857 wgs = wgp->wgp_session_unstable; 2858 if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE) 2859 /* XXX Can this happen? */ 2860 return; 2861 2862 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2863 wgs->wgs_time_established = time_uptime; 2864 wgs->wgs_time_last_data_sent = 0; 2865 wgs->wgs_is_initiator = false; 2866 WG_TRACE("WGS_STATE_ESTABLISHED"); 2867 2868 wg_swap_sessions(wgp); 2869 KASSERT(wgs == wgp->wgp_session_stable); 2870 wgs_prev = wgp->wgp_session_unstable; 2871 getnanotime(&wgp->wgp_last_handshake_time); 2872 wgp->wgp_handshake_start_time = 0; 2873 wgp->wgp_last_sent_mac1_valid = false; 2874 wgp->wgp_last_sent_cookie_valid = false; 2875 2876 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2877 /* Wait for wg_get_stable_session to drain. */ 2878 pserialize_perform(wgp->wgp_psz); 2879 2880 /* Transition ESTABLISHED->DESTROYING. */ 2881 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2882 2883 /* We can't destroy the old session immediately */ 2884 wg_schedule_session_dtor_timer(wgp); 2885 } else { 2886 KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN, 2887 "state=%d", wgs_prev->wgs_state); 2888 wg_clear_states(wgs_prev); 2889 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 2890 } 2891 2892 /* Anyway run a softint to flush pending packets */ 2893 kpreempt_disable(); 2894 softint_schedule(wgp->wgp_si); 2895 kpreempt_enable(); 2896 } 2897 2898 static void 2899 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 2900 { 2901 2902 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 2903 2904 KASSERT(mutex_owned(wgp->wgp_lock)); 2905 2906 if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) { 2907 pserialize_perform(wgp->wgp_psz); 2908 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 2909 wg_psref_class); 2910 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 2911 wg_psref_class); 2912 atomic_store_release(&wgp->wgp_endpoint_changing, 0); 2913 } 2914 } 2915 2916 static void 2917 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 2918 { 2919 struct wg_session *wgs; 2920 2921 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 2922 2923 KASSERT(mutex_owned(wgp->wgp_lock)); 2924 2925 wgs = wgp->wgp_session_stable; 2926 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) 2927 return; 2928 2929 wg_send_keepalive_msg(wgp, wgs); 2930 } 2931 2932 static void 2933 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 2934 { 2935 struct wg_session *wgs; 2936 2937 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 2938 2939 KASSERT(mutex_owned(wgp->wgp_lock)); 2940 2941 wgs = wgp->wgp_session_unstable; 2942 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 2943 wg_put_session_index(wg, wgs); 2944 } 2945 } 2946 2947 static void 2948 wg_process_peer_tasks(struct wg_softc *wg) 2949 { 2950 struct wg_peer *wgp; 2951 int s; 2952 2953 /* XXX should avoid checking all peers */ 2954 s = pserialize_read_enter(); 2955 WG_PEER_READER_FOREACH(wgp, wg) { 2956 struct psref psref; 2957 unsigned int tasks; 2958 2959 if (wgp->wgp_tasks == 0) 2960 continue; 2961 2962 wg_get_peer(wgp, &psref); 2963 pserialize_read_exit(s); 2964 2965 restart: 2966 tasks = atomic_swap_uint(&wgp->wgp_tasks, 0); 2967 KASSERT(tasks != 0); 2968 2969 WG_DLOG("tasks=%x\n", tasks); 2970 2971 mutex_enter(wgp->wgp_lock); 2972 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 2973 wg_task_send_init_message(wg, wgp); 2974 if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE)) 2975 wg_task_retry_handshake(wg, wgp); 2976 if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION)) 2977 wg_task_establish_session(wg, wgp); 2978 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 2979 wg_task_endpoint_changed(wg, wgp); 2980 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 2981 wg_task_send_keepalive_message(wg, wgp); 2982 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 2983 wg_task_destroy_prev_session(wg, wgp); 2984 mutex_exit(wgp->wgp_lock); 2985 2986 /* New tasks may be scheduled during processing tasks */ 2987 WG_DLOG("wgp_tasks=%d\n", wgp->wgp_tasks); 2988 if (wgp->wgp_tasks != 0) 2989 goto restart; 2990 2991 s = pserialize_read_enter(); 2992 wg_put_peer(wgp, &psref); 2993 } 2994 pserialize_read_exit(s); 2995 } 2996 2997 static void 2998 wg_worker(void *arg) 2999 { 3000 struct wg_softc *wg = arg; 3001 struct wg_worker *wgw = wg->wg_worker; 3002 bool todie = false; 3003 3004 KASSERT(wg != NULL); 3005 KASSERT(wgw != NULL); 3006 3007 while (!todie) { 3008 int reasons; 3009 int bound; 3010 3011 mutex_enter(&wgw->wgw_lock); 3012 /* New tasks may come during task handling */ 3013 while ((reasons = wgw->wgw_wakeup_reasons) == 0 && 3014 !(todie = wgw->wgw_todie)) 3015 cv_wait(&wgw->wgw_cv, &wgw->wgw_lock); 3016 wgw->wgw_wakeup_reasons = 0; 3017 mutex_exit(&wgw->wgw_lock); 3018 3019 bound = curlwp_bind(); 3020 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4)) 3021 wg_receive_packets(wg, AF_INET); 3022 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6)) 3023 wg_receive_packets(wg, AF_INET6); 3024 if (ISSET(reasons, WG_WAKEUP_REASON_PEER)) 3025 wg_process_peer_tasks(wg); 3026 curlwp_bindx(bound); 3027 } 3028 kthread_exit(0); 3029 } 3030 3031 static void 3032 wg_wakeup_worker(struct wg_worker *wgw, const int reason) 3033 { 3034 3035 mutex_enter(&wgw->wgw_lock); 3036 wgw->wgw_wakeup_reasons |= reason; 3037 cv_broadcast(&wgw->wgw_cv); 3038 mutex_exit(&wgw->wgw_lock); 3039 } 3040 3041 static int 3042 wg_bind_port(struct wg_softc *wg, const uint16_t port) 3043 { 3044 int error; 3045 struct wg_worker *wgw = wg->wg_worker; 3046 uint16_t old_port = wg->wg_listen_port; 3047 3048 if (port != 0 && old_port == port) 3049 return 0; 3050 3051 struct sockaddr_in _sin, *sin = &_sin; 3052 sin->sin_len = sizeof(*sin); 3053 sin->sin_family = AF_INET; 3054 sin->sin_addr.s_addr = INADDR_ANY; 3055 sin->sin_port = htons(port); 3056 3057 error = sobind(wgw->wgw_so4, sintosa(sin), curlwp); 3058 if (error != 0) 3059 return error; 3060 3061 #ifdef INET6 3062 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 3063 sin6->sin6_len = sizeof(*sin6); 3064 sin6->sin6_family = AF_INET6; 3065 sin6->sin6_addr = in6addr_any; 3066 sin6->sin6_port = htons(port); 3067 3068 error = sobind(wgw->wgw_so6, sin6tosa(sin6), curlwp); 3069 if (error != 0) 3070 return error; 3071 #endif 3072 3073 wg->wg_listen_port = port; 3074 3075 return 0; 3076 } 3077 3078 static void 3079 wg_so_upcall(struct socket *so, void *arg, int events, int waitflag) 3080 { 3081 struct wg_worker *wgw = arg; 3082 int reason; 3083 3084 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 3085 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 : 3086 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6; 3087 wg_wakeup_worker(wgw, reason); 3088 } 3089 3090 static int 3091 wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 3092 struct sockaddr *src, void *arg) 3093 { 3094 struct wg_softc *wg = arg; 3095 struct wg_msg wgm; 3096 struct mbuf *m = *mp; 3097 3098 WG_TRACE("enter"); 3099 3100 /* Verify the mbuf chain is long enough to have a wg msg header. */ 3101 KASSERT(offset <= m_length(m)); 3102 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 3103 /* drop on the floor */ 3104 m_freem(m); 3105 return -1; 3106 } 3107 3108 /* 3109 * Copy the message header (32-bit message type) out -- we'll 3110 * worry about contiguity and alignment later. 3111 */ 3112 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 3113 WG_DLOG("type=%d\n", le32toh(wgm.wgm_type)); 3114 3115 /* 3116 * Handle DATA packets promptly as they arrive. Other packets 3117 * may require expensive public-key crypto and are not as 3118 * sensitive to latency, so defer them to the worker thread. 3119 */ 3120 switch (le32toh(wgm.wgm_type)) { 3121 case WG_MSG_TYPE_DATA: 3122 /* handle immediately */ 3123 m_adj(m, offset); 3124 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 3125 m = m_pullup(m, sizeof(struct wg_msg_data)); 3126 if (m == NULL) 3127 return -1; 3128 } 3129 wg_handle_msg_data(wg, m, src); 3130 *mp = NULL; 3131 return 1; 3132 case WG_MSG_TYPE_INIT: 3133 case WG_MSG_TYPE_RESP: 3134 case WG_MSG_TYPE_COOKIE: 3135 /* pass through to so_receive in wg_receive_packets */ 3136 return 0; 3137 default: 3138 /* drop on the floor */ 3139 m_freem(m); 3140 return -1; 3141 } 3142 } 3143 3144 static int 3145 wg_worker_socreate(struct wg_softc *wg, struct wg_worker *wgw, const int af, 3146 struct socket **sop) 3147 { 3148 int error; 3149 struct socket *so; 3150 3151 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3152 if (error != 0) 3153 return error; 3154 3155 solock(so); 3156 so->so_upcallarg = wgw; 3157 so->so_upcall = wg_so_upcall; 3158 so->so_rcv.sb_flags |= SB_UPCALL; 3159 if (af == AF_INET) 3160 in_pcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3161 #if INET6 3162 else 3163 in6_pcb_register_overudp_cb(sotoin6pcb(so), wg_overudp_cb, wg); 3164 #endif 3165 sounlock(so); 3166 3167 *sop = so; 3168 3169 return 0; 3170 } 3171 3172 static int 3173 wg_worker_init(struct wg_softc *wg) 3174 { 3175 int error; 3176 struct wg_worker *wgw; 3177 const char *ifname = wg->wg_if.if_xname; 3178 struct socket *so; 3179 3180 wgw = kmem_zalloc(sizeof(*wgw), KM_SLEEP); 3181 3182 mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3183 cv_init(&wgw->wgw_cv, ifname); 3184 wgw->wgw_todie = false; 3185 wgw->wgw_wakeup_reasons = 0; 3186 3187 error = wg_worker_socreate(wg, wgw, AF_INET, &so); 3188 if (error != 0) 3189 goto error; 3190 wgw->wgw_so4 = so; 3191 #ifdef INET6 3192 error = wg_worker_socreate(wg, wgw, AF_INET6, &so); 3193 if (error != 0) 3194 goto error; 3195 wgw->wgw_so6 = so; 3196 #endif 3197 3198 wg->wg_worker = wgw; 3199 3200 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, 3201 NULL, wg_worker, wg, &wg->wg_worker_lwp, "%s", ifname); 3202 if (error != 0) 3203 goto error; 3204 3205 return 0; 3206 3207 error: 3208 #ifdef INET6 3209 if (wgw->wgw_so6 != NULL) 3210 soclose(wgw->wgw_so6); 3211 #endif 3212 if (wgw->wgw_so4 != NULL) 3213 soclose(wgw->wgw_so4); 3214 cv_destroy(&wgw->wgw_cv); 3215 mutex_destroy(&wgw->wgw_lock); 3216 3217 kmem_free(wgw, sizeof(*wgw)); 3218 3219 return error; 3220 } 3221 3222 static void 3223 wg_worker_destroy(struct wg_softc *wg) 3224 { 3225 struct wg_worker *wgw = wg->wg_worker; 3226 3227 mutex_enter(&wgw->wgw_lock); 3228 wgw->wgw_todie = true; 3229 wgw->wgw_wakeup_reasons = 0; 3230 cv_broadcast(&wgw->wgw_cv); 3231 mutex_exit(&wgw->wgw_lock); 3232 3233 kthread_join(wg->wg_worker_lwp); 3234 3235 #ifdef INET6 3236 soclose(wgw->wgw_so6); 3237 #endif 3238 soclose(wgw->wgw_so4); 3239 cv_destroy(&wgw->wgw_cv); 3240 mutex_destroy(&wgw->wgw_lock); 3241 kmem_free(wg->wg_worker, sizeof(struct wg_worker)); 3242 wg->wg_worker = NULL; 3243 } 3244 3245 static bool 3246 wg_session_hit_limits(struct wg_session *wgs) 3247 { 3248 3249 /* 3250 * [W] 6.2: Transport Message Limits 3251 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3252 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3253 * comes first, WireGuard will refuse to send any more transport data 3254 * messages using the current secure session, ..." 3255 */ 3256 KASSERT(wgs->wgs_time_established != 0); 3257 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3258 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3259 return true; 3260 } else if (wg_session_get_send_counter(wgs) > 3261 wg_reject_after_messages) { 3262 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3263 return true; 3264 } 3265 3266 return false; 3267 } 3268 3269 static void 3270 wg_peer_softint(void *arg) 3271 { 3272 struct wg_peer *wgp = arg; 3273 struct wg_session *wgs; 3274 struct mbuf *m; 3275 struct psref psref; 3276 3277 if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) { 3278 /* XXX how to treat? */ 3279 WG_TRACE("skipped"); 3280 return; 3281 } 3282 if (wg_session_hit_limits(wgs)) { 3283 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3284 goto out; 3285 } 3286 WG_TRACE("running"); 3287 3288 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3289 wg_send_data_msg(wgp, wgs, m); 3290 } 3291 out: 3292 wg_put_session(wgs, &psref); 3293 } 3294 3295 static void 3296 wg_rekey_timer(void *arg) 3297 { 3298 struct wg_peer *wgp = arg; 3299 3300 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3301 } 3302 3303 static void 3304 wg_purge_pending_packets(struct wg_peer *wgp) 3305 { 3306 struct mbuf *m; 3307 3308 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3309 m_freem(m); 3310 } 3311 } 3312 3313 static void 3314 wg_handshake_timeout_timer(void *arg) 3315 { 3316 struct wg_peer *wgp = arg; 3317 3318 WG_TRACE("enter"); 3319 3320 wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE); 3321 } 3322 3323 static struct wg_peer * 3324 wg_alloc_peer(struct wg_softc *wg) 3325 { 3326 struct wg_peer *wgp; 3327 3328 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3329 3330 wgp->wgp_sc = wg; 3331 wgp->wgp_q = pcq_create(1024, KM_SLEEP); 3332 wgp->wgp_si = softint_establish(SOFTINT_NET, wg_peer_softint, wgp); 3333 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3334 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3335 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3336 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3337 wg_handshake_timeout_timer, wgp); 3338 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3339 callout_setfunc(&wgp->wgp_session_dtor_timer, 3340 wg_session_dtor_timer, wgp); 3341 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3342 wgp->wgp_endpoint_changing = false; 3343 wgp->wgp_endpoint_available = false; 3344 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3345 wgp->wgp_psz = pserialize_create(); 3346 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3347 3348 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3349 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3350 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3351 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3352 3353 struct wg_session *wgs; 3354 wgp->wgp_session_stable = 3355 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3356 wgp->wgp_session_unstable = 3357 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3358 wgs = wgp->wgp_session_stable; 3359 wgs->wgs_peer = wgp; 3360 wgs->wgs_state = WGS_STATE_UNKNOWN; 3361 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3362 #ifndef __HAVE_ATOMIC64_LOADSTORE 3363 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3364 #endif 3365 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3366 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3367 3368 wgs = wgp->wgp_session_unstable; 3369 wgs->wgs_peer = wgp; 3370 wgs->wgs_state = WGS_STATE_UNKNOWN; 3371 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3372 #ifndef __HAVE_ATOMIC64_LOADSTORE 3373 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3374 #endif 3375 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3376 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET); 3377 3378 return wgp; 3379 } 3380 3381 static void 3382 wg_destroy_peer(struct wg_peer *wgp) 3383 { 3384 struct wg_session *wgs; 3385 struct wg_softc *wg = wgp->wgp_sc; 3386 3387 /* Prevent new packets from this peer on any source address. */ 3388 rw_enter(wg->wg_rwlock, RW_WRITER); 3389 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3390 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3391 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3392 struct radix_node *rn; 3393 3394 KASSERT(rnh != NULL); 3395 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3396 &wga->wga_sa_mask, rnh); 3397 if (rn == NULL) { 3398 char addrstr[128]; 3399 sockaddr_format(&wga->wga_sa_addr, addrstr, 3400 sizeof(addrstr)); 3401 WGLOG(LOG_WARNING, "Couldn't delete %s", addrstr); 3402 } 3403 } 3404 rw_exit(wg->wg_rwlock); 3405 3406 /* Purge pending packets. */ 3407 wg_purge_pending_packets(wgp); 3408 3409 /* Halt all packet processing and timeouts. */ 3410 softint_disestablish(wgp->wgp_si); 3411 callout_halt(&wgp->wgp_rekey_timer, NULL); 3412 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3413 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3414 3415 wgs = wgp->wgp_session_unstable; 3416 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3417 mutex_enter(wgp->wgp_lock); 3418 wg_destroy_session(wg, wgs); 3419 mutex_exit(wgp->wgp_lock); 3420 } 3421 mutex_destroy(&wgs->wgs_recvwin->lock); 3422 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3423 #ifndef __HAVE_ATOMIC64_LOADSTORE 3424 mutex_destroy(&wgs->wgs_send_counter_lock); 3425 #endif 3426 kmem_free(wgs, sizeof(*wgs)); 3427 3428 wgs = wgp->wgp_session_stable; 3429 if (wgs->wgs_state != WGS_STATE_UNKNOWN) { 3430 mutex_enter(wgp->wgp_lock); 3431 wg_destroy_session(wg, wgs); 3432 mutex_exit(wgp->wgp_lock); 3433 } 3434 mutex_destroy(&wgs->wgs_recvwin->lock); 3435 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3436 #ifndef __HAVE_ATOMIC64_LOADSTORE 3437 mutex_destroy(&wgs->wgs_send_counter_lock); 3438 #endif 3439 kmem_free(wgs, sizeof(*wgs)); 3440 3441 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3442 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3443 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3444 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3445 3446 pserialize_destroy(wgp->wgp_psz); 3447 pcq_destroy(wgp->wgp_q); 3448 mutex_obj_free(wgp->wgp_lock); 3449 3450 kmem_free(wgp, sizeof(*wgp)); 3451 } 3452 3453 static void 3454 wg_destroy_all_peers(struct wg_softc *wg) 3455 { 3456 struct wg_peer *wgp, *wgp0 __diagused; 3457 void *garbage_byname, *garbage_bypubkey; 3458 3459 restart: 3460 garbage_byname = garbage_bypubkey = NULL; 3461 mutex_enter(wg->wg_lock); 3462 WG_PEER_WRITER_FOREACH(wgp, wg) { 3463 if (wgp->wgp_name[0]) { 3464 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name, 3465 strlen(wgp->wgp_name)); 3466 KASSERT(wgp0 == wgp); 3467 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3468 } 3469 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3470 sizeof(wgp->wgp_pubkey)); 3471 KASSERT(wgp0 == wgp); 3472 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3473 WG_PEER_WRITER_REMOVE(wgp); 3474 wg->wg_npeers--; 3475 mutex_enter(wgp->wgp_lock); 3476 pserialize_perform(wgp->wgp_psz); 3477 mutex_exit(wgp->wgp_lock); 3478 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3479 break; 3480 } 3481 mutex_exit(wg->wg_lock); 3482 3483 if (wgp == NULL) 3484 return; 3485 3486 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3487 3488 wg_destroy_peer(wgp); 3489 thmap_gc(wg->wg_peers_byname, garbage_byname); 3490 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3491 3492 goto restart; 3493 } 3494 3495 static int 3496 wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3497 { 3498 struct wg_peer *wgp, *wgp0 __diagused; 3499 void *garbage_byname, *garbage_bypubkey; 3500 3501 mutex_enter(wg->wg_lock); 3502 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name)); 3503 if (wgp != NULL) { 3504 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 3505 sizeof(wgp->wgp_pubkey)); 3506 KASSERT(wgp0 == wgp); 3507 garbage_byname = thmap_stage_gc(wg->wg_peers_byname); 3508 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey); 3509 WG_PEER_WRITER_REMOVE(wgp); 3510 wg->wg_npeers--; 3511 mutex_enter(wgp->wgp_lock); 3512 pserialize_perform(wgp->wgp_psz); 3513 mutex_exit(wgp->wgp_lock); 3514 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3515 } 3516 mutex_exit(wg->wg_lock); 3517 3518 if (wgp == NULL) 3519 return ENOENT; 3520 3521 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3522 3523 wg_destroy_peer(wgp); 3524 thmap_gc(wg->wg_peers_byname, garbage_byname); 3525 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey); 3526 3527 return 0; 3528 } 3529 3530 static int 3531 wg_if_attach(struct wg_softc *wg) 3532 { 3533 int error; 3534 3535 wg->wg_if.if_addrlen = 0; 3536 wg->wg_if.if_mtu = WG_MTU; 3537 wg->wg_if.if_flags = IFF_MULTICAST; 3538 wg->wg_if.if_extflags = IFEF_NO_LINK_STATE_CHANGE; 3539 wg->wg_if.if_extflags |= IFEF_MPSAFE; 3540 wg->wg_if.if_ioctl = wg_ioctl; 3541 wg->wg_if.if_output = wg_output; 3542 wg->wg_if.if_init = wg_init; 3543 wg->wg_if.if_stop = wg_stop; 3544 wg->wg_if.if_type = IFT_OTHER; 3545 wg->wg_if.if_dlt = DLT_NULL; 3546 wg->wg_if.if_softc = wg; 3547 IFQ_SET_READY(&wg->wg_if.if_snd); 3548 3549 error = if_initialize(&wg->wg_if); 3550 if (error != 0) 3551 return error; 3552 3553 if_alloc_sadl(&wg->wg_if); 3554 if_register(&wg->wg_if); 3555 3556 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3557 3558 return 0; 3559 } 3560 3561 static int 3562 wg_clone_create(struct if_clone *ifc, int unit) 3563 { 3564 struct wg_softc *wg; 3565 int error; 3566 3567 wg = kmem_zalloc(sizeof(struct wg_softc), KM_SLEEP); 3568 3569 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3570 3571 error = wg_worker_init(wg); 3572 if (error != 0) { 3573 kmem_free(wg, sizeof(struct wg_softc)); 3574 return error; 3575 } 3576 3577 rn_inithead((void **)&wg->wg_rtable_ipv4, 3578 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3579 #ifdef INET6 3580 rn_inithead((void **)&wg->wg_rtable_ipv6, 3581 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3582 #endif 3583 3584 PSLIST_INIT(&wg->wg_peers); 3585 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY); 3586 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY); 3587 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY); 3588 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3589 wg->wg_rwlock = rw_obj_alloc(); 3590 wg->wg_ops = &wg_ops_rumpkernel; 3591 3592 error = wg_if_attach(wg); 3593 if (error != 0) { 3594 wg_worker_destroy(wg); 3595 if (wg->wg_rtable_ipv4 != NULL) 3596 free(wg->wg_rtable_ipv4, M_RTABLE); 3597 if (wg->wg_rtable_ipv6 != NULL) 3598 free(wg->wg_rtable_ipv6, M_RTABLE); 3599 PSLIST_DESTROY(&wg->wg_peers); 3600 mutex_obj_free(wg->wg_lock); 3601 thmap_destroy(wg->wg_sessions_byindex); 3602 thmap_destroy(wg->wg_peers_byname); 3603 thmap_destroy(wg->wg_peers_bypubkey); 3604 kmem_free(wg, sizeof(struct wg_softc)); 3605 return error; 3606 } 3607 3608 mutex_enter(&wg_softcs.lock); 3609 LIST_INSERT_HEAD(&wg_softcs.list, wg, wg_list); 3610 mutex_exit(&wg_softcs.lock); 3611 3612 return 0; 3613 } 3614 3615 static int 3616 wg_clone_destroy(struct ifnet *ifp) 3617 { 3618 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3619 3620 mutex_enter(&wg_softcs.lock); 3621 LIST_REMOVE(wg, wg_list); 3622 mutex_exit(&wg_softcs.lock); 3623 3624 #ifdef WG_RUMPKERNEL 3625 if (wg_user_mode(wg)) { 3626 rumpuser_wg_destroy(wg->wg_user); 3627 wg->wg_user = NULL; 3628 } 3629 #endif 3630 3631 bpf_detach(ifp); 3632 if_detach(ifp); 3633 wg_worker_destroy(wg); 3634 wg_destroy_all_peers(wg); 3635 if (wg->wg_rtable_ipv4 != NULL) 3636 free(wg->wg_rtable_ipv4, M_RTABLE); 3637 if (wg->wg_rtable_ipv6 != NULL) 3638 free(wg->wg_rtable_ipv6, M_RTABLE); 3639 3640 PSLIST_DESTROY(&wg->wg_peers); 3641 thmap_destroy(wg->wg_sessions_byindex); 3642 thmap_destroy(wg->wg_peers_byname); 3643 thmap_destroy(wg->wg_peers_bypubkey); 3644 mutex_obj_free(wg->wg_lock); 3645 rw_obj_free(wg->wg_rwlock); 3646 3647 kmem_free(wg, sizeof(struct wg_softc)); 3648 3649 return 0; 3650 } 3651 3652 static struct wg_peer * 3653 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3654 struct psref *psref) 3655 { 3656 struct radix_node_head *rnh; 3657 struct radix_node *rn; 3658 struct wg_peer *wgp = NULL; 3659 struct wg_allowedip *wga; 3660 3661 #ifdef WG_DEBUG_LOG 3662 char addrstr[128]; 3663 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3664 WG_DLOG("sa=%s\n", addrstr); 3665 #endif 3666 3667 rw_enter(wg->wg_rwlock, RW_READER); 3668 3669 rnh = wg_rnh(wg, sa->sa_family); 3670 if (rnh == NULL) 3671 goto out; 3672 3673 rn = rnh->rnh_matchaddr(sa, rnh); 3674 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3675 goto out; 3676 3677 WG_TRACE("success"); 3678 3679 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3680 wgp = wga->wga_peer; 3681 wg_get_peer(wgp, psref); 3682 3683 out: 3684 rw_exit(wg->wg_rwlock); 3685 return wgp; 3686 } 3687 3688 static void 3689 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3690 struct wg_session *wgs, struct wg_msg_data *wgmd) 3691 { 3692 3693 memset(wgmd, 0, sizeof(*wgmd)); 3694 wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA); 3695 wgmd->wgmd_receiver = wgs->wgs_remote_index; 3696 /* [W] 5.4.6: msg.counter := Nm^send */ 3697 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3698 wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs)); 3699 WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter)); 3700 } 3701 3702 static int 3703 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3704 const struct rtentry *rt) 3705 { 3706 struct wg_softc *wg = ifp->if_softc; 3707 struct wg_peer *wgp = NULL; 3708 struct wg_session *wgs = NULL; 3709 struct psref wgp_psref, wgs_psref; 3710 int bound; 3711 int error; 3712 3713 bound = curlwp_bind(); 3714 3715 /* TODO make the nest limit configurable via sysctl */ 3716 error = if_tunnel_check_nesting(ifp, m, 1); 3717 if (error) { 3718 WGLOG(LOG_ERR, "tunneling loop detected and packet dropped\n"); 3719 goto out; 3720 } 3721 3722 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3723 3724 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3725 3726 m->m_flags &= ~(M_BCAST|M_MCAST); 3727 3728 wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref); 3729 if (wgp == NULL) { 3730 WG_TRACE("peer not found"); 3731 error = EHOSTUNREACH; 3732 goto out; 3733 } 3734 3735 /* Clear checksum-offload flags. */ 3736 m->m_pkthdr.csum_flags = 0; 3737 m->m_pkthdr.csum_data = 0; 3738 3739 if (!pcq_put(wgp->wgp_q, m)) { 3740 error = ENOBUFS; 3741 goto out; 3742 } 3743 m = NULL; /* consumed */ 3744 3745 wgs = wg_get_stable_session(wgp, &wgs_psref); 3746 if (wgs != NULL && !wg_session_hit_limits(wgs)) { 3747 kpreempt_disable(); 3748 softint_schedule(wgp->wgp_si); 3749 kpreempt_enable(); 3750 WG_TRACE("softint scheduled"); 3751 } else { 3752 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3753 WG_TRACE("softint NOT scheduled"); 3754 } 3755 error = 0; 3756 3757 out: 3758 if (wgs != NULL) 3759 wg_put_session(wgs, &wgs_psref); 3760 if (wgp != NULL) 3761 wg_put_peer(wgp, &wgp_psref); 3762 if (m != NULL) 3763 m_freem(m); 3764 curlwp_bindx(bound); 3765 return error; 3766 } 3767 3768 static int 3769 wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3770 { 3771 struct psref psref; 3772 struct wg_sockaddr *wgsa; 3773 int error; 3774 struct socket *so; 3775 3776 wgsa = wg_get_endpoint_sa(wgp, &psref); 3777 so = wg_get_so_by_peer(wgp, wgsa); 3778 solock(so); 3779 if (wgsatosa(wgsa)->sa_family == AF_INET) { 3780 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 3781 } else { 3782 #ifdef INET6 3783 error = udp6_output(sotoin6pcb(so), m, wgsatosin6(wgsa), 3784 NULL, curlwp); 3785 #else 3786 m_freem(m); 3787 error = EPFNOSUPPORT; 3788 #endif 3789 } 3790 sounlock(so); 3791 wg_put_sa(wgp, wgsa, &psref); 3792 3793 return error; 3794 } 3795 3796 /* Inspired by pppoe_get_mbuf */ 3797 static struct mbuf * 3798 wg_get_mbuf(size_t leading_len, size_t len) 3799 { 3800 struct mbuf *m; 3801 3802 KASSERT(leading_len <= MCLBYTES); 3803 KASSERT(len <= MCLBYTES - leading_len); 3804 3805 m = m_gethdr(M_DONTWAIT, MT_DATA); 3806 if (m == NULL) 3807 return NULL; 3808 if (len + leading_len > MHLEN) { 3809 m_clget(m, M_DONTWAIT); 3810 if ((m->m_flags & M_EXT) == 0) { 3811 m_free(m); 3812 return NULL; 3813 } 3814 } 3815 m->m_data += leading_len; 3816 m->m_pkthdr.len = m->m_len = len; 3817 3818 return m; 3819 } 3820 3821 static int 3822 wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 3823 struct mbuf *m) 3824 { 3825 struct wg_softc *wg = wgp->wgp_sc; 3826 int error; 3827 size_t inner_len, padded_len, encrypted_len; 3828 char *padded_buf = NULL; 3829 size_t mlen; 3830 struct wg_msg_data *wgmd; 3831 bool free_padded_buf = false; 3832 struct mbuf *n; 3833 size_t leading_len = max_linkhdr + sizeof(struct ip6_hdr) + 3834 sizeof(struct udphdr); 3835 3836 mlen = m_length(m); 3837 inner_len = mlen; 3838 padded_len = roundup(mlen, 16); 3839 encrypted_len = padded_len + WG_AUTHTAG_LEN; 3840 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 3841 inner_len, padded_len, encrypted_len); 3842 if (mlen != 0) { 3843 bool success; 3844 success = m_ensure_contig(&m, padded_len); 3845 if (success) { 3846 padded_buf = mtod(m, char *); 3847 } else { 3848 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 3849 if (padded_buf == NULL) { 3850 error = ENOBUFS; 3851 goto end; 3852 } 3853 free_padded_buf = true; 3854 m_copydata(m, 0, mlen, padded_buf); 3855 } 3856 memset(padded_buf + mlen, 0, padded_len - inner_len); 3857 } 3858 3859 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 3860 if (n == NULL) { 3861 error = ENOBUFS; 3862 goto end; 3863 } 3864 KASSERT(n->m_len >= sizeof(*wgmd)); 3865 wgmd = mtod(n, struct wg_msg_data *); 3866 wg_fill_msg_data(wg, wgp, wgs, wgmd); 3867 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 3868 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 3869 wgs->wgs_tkey_send, le64toh(wgmd->wgmd_counter), 3870 padded_buf, padded_len, 3871 NULL, 0); 3872 3873 error = wg->wg_ops->send_data_msg(wgp, n); 3874 if (error == 0) { 3875 struct ifnet *ifp = &wg->wg_if; 3876 if_statadd(ifp, if_obytes, mlen); 3877 if_statinc(ifp, if_opackets); 3878 if (wgs->wgs_is_initiator && 3879 wgs->wgs_time_last_data_sent == 0) { 3880 /* 3881 * [W] 6.2 Transport Message Limits 3882 * "if a peer is the initiator of a current secure 3883 * session, WireGuard will send a handshake initiation 3884 * message to begin a new secure session if, after 3885 * transmitting a transport data message, the current 3886 * secure session is REKEY-AFTER-TIME seconds old," 3887 */ 3888 wg_schedule_rekey_timer(wgp); 3889 } 3890 wgs->wgs_time_last_data_sent = time_uptime; 3891 if (wg_session_get_send_counter(wgs) >= 3892 wg_rekey_after_messages) { 3893 /* 3894 * [W] 6.2 Transport Message Limits 3895 * "WireGuard will try to create a new session, by 3896 * sending a handshake initiation message (section 3897 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 3898 * transport data messages..." 3899 */ 3900 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3901 } 3902 } 3903 end: 3904 m_freem(m); 3905 if (free_padded_buf) 3906 kmem_intr_free(padded_buf, padded_len); 3907 return error; 3908 } 3909 3910 static void 3911 wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 3912 { 3913 pktqueue_t *pktq; 3914 size_t pktlen; 3915 3916 KASSERT(af == AF_INET || af == AF_INET6); 3917 3918 WG_TRACE(""); 3919 3920 m_set_rcvif(m, ifp); 3921 pktlen = m->m_pkthdr.len; 3922 3923 bpf_mtap_af(ifp, af, m, BPF_D_IN); 3924 3925 switch (af) { 3926 case AF_INET: 3927 pktq = ip_pktq; 3928 break; 3929 #ifdef INET6 3930 case AF_INET6: 3931 pktq = ip6_pktq; 3932 break; 3933 #endif 3934 default: 3935 panic("invalid af=%d", af); 3936 } 3937 3938 const u_int h = curcpu()->ci_index; 3939 if (__predict_true(pktq_enqueue(pktq, m, h))) { 3940 if_statadd(ifp, if_ibytes, pktlen); 3941 if_statinc(ifp, if_ipackets); 3942 } else { 3943 m_freem(m); 3944 } 3945 } 3946 3947 static void 3948 wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 3949 const uint8_t privkey[WG_STATIC_KEY_LEN]) 3950 { 3951 3952 crypto_scalarmult_base(pubkey, privkey); 3953 } 3954 3955 static int 3956 wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 3957 { 3958 struct radix_node_head *rnh; 3959 struct radix_node *rn; 3960 int error = 0; 3961 3962 rw_enter(wg->wg_rwlock, RW_WRITER); 3963 rnh = wg_rnh(wg, wga->wga_family); 3964 KASSERT(rnh != NULL); 3965 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 3966 wga->wga_nodes); 3967 rw_exit(wg->wg_rwlock); 3968 3969 if (rn == NULL) 3970 error = EEXIST; 3971 3972 return error; 3973 } 3974 3975 static int 3976 wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 3977 struct wg_peer **wgpp) 3978 { 3979 int error = 0; 3980 const void *pubkey; 3981 size_t pubkey_len; 3982 const void *psk; 3983 size_t psk_len; 3984 const char *name = NULL; 3985 3986 if (prop_dictionary_get_string(peer, "name", &name)) { 3987 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 3988 error = EINVAL; 3989 goto out; 3990 } 3991 } 3992 3993 if (!prop_dictionary_get_data(peer, "public_key", 3994 &pubkey, &pubkey_len)) { 3995 error = EINVAL; 3996 goto out; 3997 } 3998 #ifdef WG_DEBUG_DUMP 3999 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n", pubkey, pubkey_len); 4000 for (int _i = 0; _i < pubkey_len; _i++) 4001 log(LOG_DEBUG, "%c", ((const char *)pubkey)[_i]); 4002 log(LOG_DEBUG, "\n"); 4003 #endif 4004 4005 struct wg_peer *wgp = wg_alloc_peer(wg); 4006 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 4007 if (name != NULL) 4008 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 4009 4010 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 4011 if (psk_len != sizeof(wgp->wgp_psk)) { 4012 error = EINVAL; 4013 goto out; 4014 } 4015 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 4016 } 4017 4018 const void *addr; 4019 size_t addr_len; 4020 struct wg_sockaddr *wgsa = wgp->wgp_endpoint; 4021 4022 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 4023 goto skip_endpoint; 4024 if (addr_len < sizeof(*wgsatosa(wgsa)) || 4025 addr_len > sizeof(*wgsatoss(wgsa))) { 4026 error = EINVAL; 4027 goto out; 4028 } 4029 memcpy(wgsatoss(wgsa), addr, addr_len); 4030 switch (wgsa_family(wgsa)) { 4031 case AF_INET: 4032 #ifdef INET6 4033 case AF_INET6: 4034 #endif 4035 break; 4036 default: 4037 error = EPFNOSUPPORT; 4038 goto out; 4039 } 4040 if (addr_len != sockaddr_getsize_by_family(wgsa_family(wgsa))) { 4041 error = EINVAL; 4042 goto out; 4043 } 4044 { 4045 char addrstr[128]; 4046 sockaddr_format(wgsatosa(wgsa), addrstr, sizeof(addrstr)); 4047 WG_DLOG("addr=%s\n", addrstr); 4048 } 4049 wgp->wgp_endpoint_available = true; 4050 4051 prop_array_t allowedips; 4052 skip_endpoint: 4053 allowedips = prop_dictionary_get(peer, "allowedips"); 4054 if (allowedips == NULL) 4055 goto skip; 4056 4057 prop_object_iterator_t _it = prop_array_iterator(allowedips); 4058 prop_dictionary_t prop_allowedip; 4059 int j = 0; 4060 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 4061 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4062 4063 if (!prop_dictionary_get_int(prop_allowedip, "family", 4064 &wga->wga_family)) 4065 continue; 4066 if (!prop_dictionary_get_data(prop_allowedip, "ip", 4067 &addr, &addr_len)) 4068 continue; 4069 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 4070 &wga->wga_cidr)) 4071 continue; 4072 4073 switch (wga->wga_family) { 4074 case AF_INET: { 4075 struct sockaddr_in sin; 4076 char addrstr[128]; 4077 struct in_addr mask; 4078 struct sockaddr_in sin_mask; 4079 4080 if (addr_len != sizeof(struct in_addr)) 4081 return EINVAL; 4082 memcpy(&wga->wga_addr4, addr, addr_len); 4083 4084 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4085 0); 4086 sockaddr_copy(&wga->wga_sa_addr, 4087 sizeof(sin), sintosa(&sin)); 4088 4089 sockaddr_format(sintosa(&sin), 4090 addrstr, sizeof(addrstr)); 4091 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4092 4093 in_len2mask(&mask, wga->wga_cidr); 4094 sockaddr_in_init(&sin_mask, &mask, 0); 4095 sockaddr_copy(&wga->wga_sa_mask, 4096 sizeof(sin_mask), sintosa(&sin_mask)); 4097 4098 break; 4099 } 4100 #ifdef INET6 4101 case AF_INET6: { 4102 struct sockaddr_in6 sin6; 4103 char addrstr[128]; 4104 struct in6_addr mask; 4105 struct sockaddr_in6 sin6_mask; 4106 4107 if (addr_len != sizeof(struct in6_addr)) 4108 return EINVAL; 4109 memcpy(&wga->wga_addr6, addr, addr_len); 4110 4111 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4112 0, 0, 0); 4113 sockaddr_copy(&wga->wga_sa_addr, 4114 sizeof(sin6), sin6tosa(&sin6)); 4115 4116 sockaddr_format(sin6tosa(&sin6), 4117 addrstr, sizeof(addrstr)); 4118 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4119 4120 in6_prefixlen2mask(&mask, wga->wga_cidr); 4121 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4122 sockaddr_copy(&wga->wga_sa_mask, 4123 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4124 4125 break; 4126 } 4127 #endif 4128 default: 4129 error = EINVAL; 4130 goto out; 4131 } 4132 wga->wga_peer = wgp; 4133 4134 error = wg_rtable_add_route(wg, wga); 4135 if (error != 0) 4136 goto out; 4137 4138 j++; 4139 } 4140 wgp->wgp_n_allowedips = j; 4141 skip: 4142 *wgpp = wgp; 4143 out: 4144 return error; 4145 } 4146 4147 static int 4148 wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4149 { 4150 int error; 4151 char *buf; 4152 4153 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4154 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4155 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4156 if (error != 0) 4157 return error; 4158 buf[ifd->ifd_len] = '\0'; 4159 #ifdef WG_DEBUG_DUMP 4160 for (int i = 0; i < ifd->ifd_len; i++) 4161 log(LOG_DEBUG, "%c", buf[i]); 4162 log(LOG_DEBUG, "\n"); 4163 #endif 4164 *_buf = buf; 4165 return 0; 4166 } 4167 4168 static int 4169 wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4170 { 4171 int error; 4172 prop_dictionary_t prop_dict; 4173 char *buf = NULL; 4174 const void *privkey; 4175 size_t privkey_len; 4176 4177 error = wg_alloc_prop_buf(&buf, ifd); 4178 if (error != 0) 4179 return error; 4180 error = EINVAL; 4181 prop_dict = prop_dictionary_internalize(buf); 4182 if (prop_dict == NULL) 4183 goto out; 4184 if (!prop_dictionary_get_data(prop_dict, "private_key", 4185 &privkey, &privkey_len)) 4186 goto out; 4187 #ifdef WG_DEBUG_DUMP 4188 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n", privkey, privkey_len); 4189 for (int i = 0; i < privkey_len; i++) 4190 log(LOG_DEBUG, "%c", ((const char *)privkey)[i]); 4191 log(LOG_DEBUG, "\n"); 4192 #endif 4193 if (privkey_len != WG_STATIC_KEY_LEN) 4194 goto out; 4195 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4196 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4197 error = 0; 4198 4199 out: 4200 kmem_free(buf, ifd->ifd_len + 1); 4201 return error; 4202 } 4203 4204 static int 4205 wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4206 { 4207 int error; 4208 prop_dictionary_t prop_dict; 4209 char *buf = NULL; 4210 uint16_t port; 4211 4212 error = wg_alloc_prop_buf(&buf, ifd); 4213 if (error != 0) 4214 return error; 4215 error = EINVAL; 4216 prop_dict = prop_dictionary_internalize(buf); 4217 if (prop_dict == NULL) 4218 goto out; 4219 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4220 goto out; 4221 4222 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4223 4224 out: 4225 kmem_free(buf, ifd->ifd_len + 1); 4226 return error; 4227 } 4228 4229 static int 4230 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4231 { 4232 int error; 4233 prop_dictionary_t prop_dict; 4234 char *buf = NULL; 4235 struct wg_peer *wgp = NULL, *wgp0 __diagused; 4236 4237 error = wg_alloc_prop_buf(&buf, ifd); 4238 if (error != 0) 4239 return error; 4240 error = EINVAL; 4241 prop_dict = prop_dictionary_internalize(buf); 4242 if (prop_dict == NULL) 4243 goto out; 4244 4245 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4246 if (error != 0) 4247 goto out; 4248 4249 mutex_enter(wg->wg_lock); 4250 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4251 sizeof(wgp->wgp_pubkey)) != NULL || 4252 (wgp->wgp_name[0] && 4253 thmap_get(wg->wg_peers_byname, wgp->wgp_name, 4254 strlen(wgp->wgp_name)) != NULL)) { 4255 mutex_exit(wg->wg_lock); 4256 wg_destroy_peer(wgp); 4257 error = EEXIST; 4258 goto out; 4259 } 4260 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey, 4261 sizeof(wgp->wgp_pubkey), wgp); 4262 KASSERT(wgp0 == wgp); 4263 if (wgp->wgp_name[0]) { 4264 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name, 4265 strlen(wgp->wgp_name), wgp); 4266 KASSERT(wgp0 == wgp); 4267 } 4268 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4269 wg->wg_npeers++; 4270 mutex_exit(wg->wg_lock); 4271 4272 out: 4273 kmem_free(buf, ifd->ifd_len + 1); 4274 return error; 4275 } 4276 4277 static int 4278 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4279 { 4280 int error; 4281 prop_dictionary_t prop_dict; 4282 char *buf = NULL; 4283 const char *name; 4284 4285 error = wg_alloc_prop_buf(&buf, ifd); 4286 if (error != 0) 4287 return error; 4288 error = EINVAL; 4289 prop_dict = prop_dictionary_internalize(buf); 4290 if (prop_dict == NULL) 4291 goto out; 4292 4293 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4294 goto out; 4295 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4296 goto out; 4297 4298 error = wg_destroy_peer_name(wg, name); 4299 out: 4300 kmem_free(buf, ifd->ifd_len + 1); 4301 return error; 4302 } 4303 4304 static int 4305 wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4306 { 4307 int error = ENOMEM; 4308 prop_dictionary_t prop_dict; 4309 prop_array_t peers = NULL; 4310 char *buf; 4311 struct wg_peer *wgp; 4312 int s, i; 4313 4314 prop_dict = prop_dictionary_create(); 4315 if (prop_dict == NULL) 4316 goto error; 4317 4318 if (!prop_dictionary_set_data(prop_dict, "private_key", wg->wg_privkey, 4319 WG_STATIC_KEY_LEN)) 4320 goto error; 4321 4322 if (wg->wg_listen_port != 0) { 4323 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4324 wg->wg_listen_port)) 4325 goto error; 4326 } 4327 4328 if (wg->wg_npeers == 0) 4329 goto skip_peers; 4330 4331 peers = prop_array_create(); 4332 if (peers == NULL) 4333 goto error; 4334 4335 s = pserialize_read_enter(); 4336 i = 0; 4337 WG_PEER_READER_FOREACH(wgp, wg) { 4338 struct wg_sockaddr *wgsa; 4339 struct psref wgp_psref, wgsa_psref; 4340 prop_dictionary_t prop_peer; 4341 4342 wg_get_peer(wgp, &wgp_psref); 4343 pserialize_read_exit(s); 4344 4345 prop_peer = prop_dictionary_create(); 4346 if (prop_peer == NULL) 4347 goto next; 4348 4349 if (strlen(wgp->wgp_name) > 0) { 4350 if (!prop_dictionary_set_string(prop_peer, "name", 4351 wgp->wgp_name)) 4352 goto next; 4353 } 4354 4355 if (!prop_dictionary_set_data(prop_peer, "public_key", 4356 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4357 goto next; 4358 4359 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4360 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4361 sizeof(wgp->wgp_psk))) { 4362 if (!prop_dictionary_set_data(prop_peer, 4363 "preshared_key", 4364 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4365 goto next; 4366 } 4367 4368 wgsa = wg_get_endpoint_sa(wgp, &wgsa_psref); 4369 CTASSERT(AF_UNSPEC == 0); 4370 if (wgsa_family(wgsa) != 0 /*AF_UNSPEC*/ && 4371 !prop_dictionary_set_data(prop_peer, "endpoint", 4372 wgsatoss(wgsa), 4373 sockaddr_getsize_by_family(wgsa_family(wgsa)))) { 4374 wg_put_sa(wgp, wgsa, &wgsa_psref); 4375 goto next; 4376 } 4377 wg_put_sa(wgp, wgsa, &wgsa_psref); 4378 4379 const struct timespec *t = &wgp->wgp_last_handshake_time; 4380 4381 if (!prop_dictionary_set_uint64(prop_peer, 4382 "last_handshake_time_sec", t->tv_sec)) 4383 goto next; 4384 if (!prop_dictionary_set_uint32(prop_peer, 4385 "last_handshake_time_nsec", t->tv_nsec)) 4386 goto next; 4387 4388 if (wgp->wgp_n_allowedips == 0) 4389 goto skip_allowedips; 4390 4391 prop_array_t allowedips = prop_array_create(); 4392 if (allowedips == NULL) 4393 goto next; 4394 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4395 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4396 prop_dictionary_t prop_allowedip; 4397 4398 prop_allowedip = prop_dictionary_create(); 4399 if (prop_allowedip == NULL) 4400 break; 4401 4402 if (!prop_dictionary_set_int(prop_allowedip, "family", 4403 wga->wga_family)) 4404 goto _next; 4405 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4406 wga->wga_cidr)) 4407 goto _next; 4408 4409 switch (wga->wga_family) { 4410 case AF_INET: 4411 if (!prop_dictionary_set_data(prop_allowedip, 4412 "ip", &wga->wga_addr4, 4413 sizeof(wga->wga_addr4))) 4414 goto _next; 4415 break; 4416 #ifdef INET6 4417 case AF_INET6: 4418 if (!prop_dictionary_set_data(prop_allowedip, 4419 "ip", &wga->wga_addr6, 4420 sizeof(wga->wga_addr6))) 4421 goto _next; 4422 break; 4423 #endif 4424 default: 4425 break; 4426 } 4427 prop_array_set(allowedips, j, prop_allowedip); 4428 _next: 4429 prop_object_release(prop_allowedip); 4430 } 4431 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4432 prop_object_release(allowedips); 4433 4434 skip_allowedips: 4435 4436 prop_array_set(peers, i, prop_peer); 4437 next: 4438 if (prop_peer) 4439 prop_object_release(prop_peer); 4440 i++; 4441 4442 s = pserialize_read_enter(); 4443 wg_put_peer(wgp, &wgp_psref); 4444 } 4445 pserialize_read_exit(s); 4446 4447 prop_dictionary_set(prop_dict, "peers", peers); 4448 prop_object_release(peers); 4449 peers = NULL; 4450 4451 skip_peers: 4452 buf = prop_dictionary_externalize(prop_dict); 4453 if (buf == NULL) 4454 goto error; 4455 if (ifd->ifd_len < (strlen(buf) + 1)) { 4456 error = EINVAL; 4457 goto error; 4458 } 4459 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4460 4461 free(buf, 0); 4462 error: 4463 if (peers != NULL) 4464 prop_object_release(peers); 4465 if (prop_dict != NULL) 4466 prop_object_release(prop_dict); 4467 4468 return error; 4469 } 4470 4471 static int 4472 wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4473 { 4474 struct wg_softc *wg = ifp->if_softc; 4475 struct ifreq *ifr = data; 4476 struct ifaddr *ifa = data; 4477 struct ifdrv *ifd = data; 4478 int error = 0; 4479 4480 switch (cmd) { 4481 case SIOCINITIFADDR: 4482 if (ifa->ifa_addr->sa_family != AF_LINK && 4483 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4484 (IFF_UP | IFF_RUNNING)) { 4485 ifp->if_flags |= IFF_UP; 4486 error = ifp->if_init(ifp); 4487 } 4488 return error; 4489 case SIOCADDMULTI: 4490 case SIOCDELMULTI: 4491 switch (ifr->ifr_addr.sa_family) { 4492 case AF_INET: /* IP supports Multicast */ 4493 break; 4494 #ifdef INET6 4495 case AF_INET6: /* IP6 supports Multicast */ 4496 break; 4497 #endif 4498 default: /* Other protocols doesn't support Multicast */ 4499 error = EAFNOSUPPORT; 4500 break; 4501 } 4502 return error; 4503 case SIOCSDRVSPEC: 4504 switch (ifd->ifd_cmd) { 4505 case WG_IOCTL_SET_PRIVATE_KEY: 4506 error = wg_ioctl_set_private_key(wg, ifd); 4507 break; 4508 case WG_IOCTL_SET_LISTEN_PORT: 4509 error = wg_ioctl_set_listen_port(wg, ifd); 4510 break; 4511 case WG_IOCTL_ADD_PEER: 4512 error = wg_ioctl_add_peer(wg, ifd); 4513 break; 4514 case WG_IOCTL_DELETE_PEER: 4515 error = wg_ioctl_delete_peer(wg, ifd); 4516 break; 4517 default: 4518 error = EINVAL; 4519 break; 4520 } 4521 return error; 4522 case SIOCGDRVSPEC: 4523 return wg_ioctl_get(wg, ifd); 4524 case SIOCSIFFLAGS: 4525 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4526 break; 4527 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4528 case IFF_RUNNING: 4529 /* 4530 * If interface is marked down and it is running, 4531 * then stop and disable it. 4532 */ 4533 (*ifp->if_stop)(ifp, 1); 4534 break; 4535 case IFF_UP: 4536 /* 4537 * If interface is marked up and it is stopped, then 4538 * start it. 4539 */ 4540 error = (*ifp->if_init)(ifp); 4541 break; 4542 default: 4543 break; 4544 } 4545 return error; 4546 #ifdef WG_RUMPKERNEL 4547 case SIOCSLINKSTR: 4548 error = wg_ioctl_linkstr(wg, ifd); 4549 if (error == 0) 4550 wg->wg_ops = &wg_ops_rumpuser; 4551 return error; 4552 #endif 4553 default: 4554 break; 4555 } 4556 4557 error = ifioctl_common(ifp, cmd, data); 4558 4559 #ifdef WG_RUMPKERNEL 4560 if (!wg_user_mode(wg)) 4561 return error; 4562 4563 /* Do the same to the corresponding tun device on the host */ 4564 /* 4565 * XXX Actually the command has not been handled yet. It 4566 * will be handled via pr_ioctl form doifioctl later. 4567 */ 4568 switch (cmd) { 4569 case SIOCAIFADDR: 4570 case SIOCDIFADDR: { 4571 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4572 struct in_aliasreq *ifra = &_ifra; 4573 KASSERT(error == ENOTTY); 4574 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4575 IFNAMSIZ); 4576 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4577 if (error == 0) 4578 error = ENOTTY; 4579 break; 4580 } 4581 #ifdef INET6 4582 case SIOCAIFADDR_IN6: 4583 case SIOCDIFADDR_IN6: { 4584 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4585 struct in6_aliasreq *ifra = &_ifra; 4586 KASSERT(error == ENOTTY); 4587 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4588 IFNAMSIZ); 4589 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4590 if (error == 0) 4591 error = ENOTTY; 4592 break; 4593 } 4594 #endif 4595 } 4596 #endif /* WG_RUMPKERNEL */ 4597 4598 return error; 4599 } 4600 4601 static int 4602 wg_init(struct ifnet *ifp) 4603 { 4604 4605 ifp->if_flags |= IFF_RUNNING; 4606 4607 /* TODO flush pending packets. */ 4608 return 0; 4609 } 4610 4611 static void 4612 wg_stop(struct ifnet *ifp, int disable) 4613 { 4614 4615 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4616 ifp->if_flags &= ~IFF_RUNNING; 4617 4618 /* Need to do something? */ 4619 } 4620 4621 #ifdef WG_DEBUG_PARAMS 4622 SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4623 { 4624 const struct sysctlnode *node = NULL; 4625 4626 sysctl_createv(clog, 0, NULL, &node, 4627 CTLFLAG_PERMANENT, 4628 CTLTYPE_NODE, "wg", 4629 SYSCTL_DESCR("wg(4)"), 4630 NULL, 0, NULL, 0, 4631 CTL_NET, CTL_CREATE, CTL_EOL); 4632 sysctl_createv(clog, 0, &node, NULL, 4633 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4634 CTLTYPE_QUAD, "rekey_after_messages", 4635 SYSCTL_DESCR("session liftime by messages"), 4636 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4637 sysctl_createv(clog, 0, &node, NULL, 4638 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4639 CTLTYPE_INT, "rekey_after_time", 4640 SYSCTL_DESCR("session liftime"), 4641 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4642 sysctl_createv(clog, 0, &node, NULL, 4643 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4644 CTLTYPE_INT, "rekey_timeout", 4645 SYSCTL_DESCR("session handshake retry time"), 4646 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4647 sysctl_createv(clog, 0, &node, NULL, 4648 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4649 CTLTYPE_INT, "rekey_attempt_time", 4650 SYSCTL_DESCR("session handshake timeout"), 4651 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4652 sysctl_createv(clog, 0, &node, NULL, 4653 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4654 CTLTYPE_INT, "keepalive_timeout", 4655 SYSCTL_DESCR("keepalive timeout"), 4656 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4657 sysctl_createv(clog, 0, &node, NULL, 4658 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4659 CTLTYPE_BOOL, "force_underload", 4660 SYSCTL_DESCR("force to detemine under load"), 4661 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4662 } 4663 #endif 4664 4665 #ifdef WG_RUMPKERNEL 4666 static bool 4667 wg_user_mode(struct wg_softc *wg) 4668 { 4669 4670 return wg->wg_user != NULL; 4671 } 4672 4673 static int 4674 wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4675 { 4676 struct ifnet *ifp = &wg->wg_if; 4677 int error; 4678 4679 if (ifp->if_flags & IFF_UP) 4680 return EBUSY; 4681 4682 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4683 /* XXX do nothing */ 4684 return 0; 4685 } else if (ifd->ifd_cmd != 0) { 4686 return EINVAL; 4687 } else if (wg->wg_user != NULL) { 4688 return EBUSY; 4689 } 4690 4691 /* Assume \0 included */ 4692 if (ifd->ifd_len > IFNAMSIZ) { 4693 return E2BIG; 4694 } else if (ifd->ifd_len < 1) { 4695 return EINVAL; 4696 } 4697 4698 char tun_name[IFNAMSIZ]; 4699 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4700 if (error != 0) 4701 return error; 4702 4703 if (strncmp(tun_name, "tun", 3) != 0) 4704 return EINVAL; 4705 4706 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 4707 4708 return error; 4709 } 4710 4711 static int 4712 wg_send_user(struct wg_peer *wgp, struct mbuf *m) 4713 { 4714 int error; 4715 struct psref psref; 4716 struct wg_sockaddr *wgsa; 4717 struct wg_softc *wg = wgp->wgp_sc; 4718 struct iovec iov[1]; 4719 4720 wgsa = wg_get_endpoint_sa(wgp, &psref); 4721 4722 iov[0].iov_base = mtod(m, void *); 4723 iov[0].iov_len = m->m_len; 4724 4725 /* Send messages to a peer via an ordinary socket. */ 4726 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 4727 4728 wg_put_sa(wgp, wgsa, &psref); 4729 4730 m_freem(m); 4731 4732 return error; 4733 } 4734 4735 static void 4736 wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 4737 { 4738 struct wg_softc *wg = ifp->if_softc; 4739 struct iovec iov[2]; 4740 struct sockaddr_storage ss; 4741 4742 KASSERT(af == AF_INET || af == AF_INET6); 4743 4744 WG_TRACE(""); 4745 4746 if (af == AF_INET) { 4747 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 4748 struct ip *ip; 4749 4750 KASSERT(m->m_len >= sizeof(struct ip)); 4751 ip = mtod(m, struct ip *); 4752 sockaddr_in_init(sin, &ip->ip_dst, 0); 4753 } else { 4754 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 4755 struct ip6_hdr *ip6; 4756 4757 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 4758 ip6 = mtod(m, struct ip6_hdr *); 4759 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 4760 } 4761 4762 iov[0].iov_base = &ss; 4763 iov[0].iov_len = ss.ss_len; 4764 iov[1].iov_base = mtod(m, void *); 4765 iov[1].iov_len = m->m_len; 4766 4767 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4768 4769 /* Send decrypted packets to users via a tun. */ 4770 rumpuser_wg_send_user(wg->wg_user, iov, 2); 4771 4772 m_freem(m); 4773 } 4774 4775 static int 4776 wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 4777 { 4778 int error; 4779 uint16_t old_port = wg->wg_listen_port; 4780 4781 if (port != 0 && old_port == port) 4782 return 0; 4783 4784 error = rumpuser_wg_sock_bind(wg->wg_user, port); 4785 if (error == 0) 4786 wg->wg_listen_port = port; 4787 return error; 4788 } 4789 4790 /* 4791 * Receive user packets. 4792 */ 4793 void 4794 rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4795 { 4796 struct ifnet *ifp = &wg->wg_if; 4797 struct mbuf *m; 4798 const struct sockaddr *dst; 4799 4800 WG_TRACE(""); 4801 4802 dst = iov[0].iov_base; 4803 4804 m = m_gethdr(M_DONTWAIT, MT_DATA); 4805 if (m == NULL) 4806 return; 4807 m->m_len = m->m_pkthdr.len = 0; 4808 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4809 4810 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4811 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4812 4813 (void)wg_output(ifp, m, dst, NULL); 4814 } 4815 4816 /* 4817 * Receive packets from a peer. 4818 */ 4819 void 4820 rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4821 { 4822 struct mbuf *m; 4823 const struct sockaddr *src; 4824 4825 WG_TRACE(""); 4826 4827 src = iov[0].iov_base; 4828 4829 m = m_gethdr(M_DONTWAIT, MT_DATA); 4830 if (m == NULL) 4831 return; 4832 m->m_len = m->m_pkthdr.len = 0; 4833 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4834 4835 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4836 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4837 4838 wg_handle_packet(wg, m, src); 4839 } 4840 #endif /* WG_RUMPKERNEL */ 4841 4842 /* 4843 * Module infrastructure 4844 */ 4845 #include "if_module.h" 4846 4847 IF_MODULE(MODULE_CLASS_DRIVER, wg, "") 4848