1*f08653c5Smvs /* $OpenBSD: if_wg.c,v 1.40 2025/01/25 14:51:34 mvs Exp $ */ 23ea0ed4dSjasper 358360b13Sdlg /* 458360b13Sdlg * Copyright (C) 2015-2020 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. 558360b13Sdlg * Copyright (C) 2019-2020 Matt Dunwoodie <ncon@noconroy.net> 658360b13Sdlg * 758360b13Sdlg * Permission to use, copy, modify, and distribute this software for any 858360b13Sdlg * purpose with or without fee is hereby granted, provided that the above 958360b13Sdlg * copyright notice and this permission notice appear in all copies. 1058360b13Sdlg * 1158360b13Sdlg * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 1258360b13Sdlg * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1358360b13Sdlg * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1458360b13Sdlg * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1558360b13Sdlg * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1658360b13Sdlg * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1758360b13Sdlg * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1858360b13Sdlg */ 1958360b13Sdlg 2058360b13Sdlg #include "bpfilter.h" 21e3884aa0Stb #include "pf.h" 2258360b13Sdlg 2358360b13Sdlg #include <sys/types.h> 2458360b13Sdlg #include <sys/systm.h> 2558360b13Sdlg #include <sys/param.h> 2658360b13Sdlg #include <sys/pool.h> 2758360b13Sdlg 2858360b13Sdlg #include <sys/socket.h> 2958360b13Sdlg #include <sys/socketvar.h> 3058360b13Sdlg #include <sys/percpu.h> 3158360b13Sdlg #include <sys/ioctl.h> 3258360b13Sdlg #include <sys/mbuf.h> 33*f08653c5Smvs #include <sys/syslog.h> 3458360b13Sdlg 3558360b13Sdlg #include <net/if.h> 3658360b13Sdlg #include <net/if_var.h> 3758360b13Sdlg #include <net/if_types.h> 3858360b13Sdlg #include <net/if_wg.h> 3958360b13Sdlg 4058360b13Sdlg #include <net/wg_noise.h> 4158360b13Sdlg #include <net/wg_cookie.h> 4258360b13Sdlg 4358360b13Sdlg #include <net/pfvar.h> 4458360b13Sdlg #include <net/route.h> 4558360b13Sdlg #include <net/bpf.h> 4691defc8bSclaudio #include <net/art.h> 4758360b13Sdlg 4858360b13Sdlg #include <netinet/ip.h> 4958360b13Sdlg #include <netinet/ip6.h> 5058360b13Sdlg #include <netinet/udp.h> 5158360b13Sdlg #include <netinet/in_pcb.h> 5258360b13Sdlg 5358360b13Sdlg #include <crypto/siphash.h> 5458360b13Sdlg 5558360b13Sdlg #define DEFAULT_MTU 1420 5658360b13Sdlg 5758360b13Sdlg #define MAX_STAGED_PKT 128 5858360b13Sdlg #define MAX_QUEUED_PKT 1024 5958360b13Sdlg #define MAX_QUEUED_PKT_MASK (MAX_QUEUED_PKT - 1) 6058360b13Sdlg 6158360b13Sdlg #define MAX_QUEUED_HANDSHAKES 4096 6258360b13Sdlg 6358360b13Sdlg #define HASHTABLE_PEER_SIZE (1 << 11) 6458360b13Sdlg #define HASHTABLE_INDEX_SIZE (1 << 13) 6558360b13Sdlg #define MAX_PEERS_PER_IFACE (1 << 20) 6658360b13Sdlg 6758360b13Sdlg #define REKEY_TIMEOUT 5 6858360b13Sdlg #define REKEY_TIMEOUT_JITTER 334 /* 1/3 sec, round for arc4random_uniform */ 6958360b13Sdlg #define KEEPALIVE_TIMEOUT 10 7058360b13Sdlg #define MAX_TIMER_HANDSHAKES (90 / REKEY_TIMEOUT) 7158360b13Sdlg #define NEW_HANDSHAKE_TIMEOUT (REKEY_TIMEOUT + KEEPALIVE_TIMEOUT) 7258360b13Sdlg #define UNDERLOAD_TIMEOUT 1 7358360b13Sdlg 74*f08653c5Smvs #define WGPRINTF(loglevel, sc, mtx, fmt, ...) do { \ 75*f08653c5Smvs if (ISSET((sc)->sc_if.if_flags, IFF_DEBUG)) { \ 76*f08653c5Smvs if (mtx) \ 77*f08653c5Smvs mtx_enter(mtx); \ 78*f08653c5Smvs log(loglevel, "%s: " fmt, (sc)->sc_if.if_xname, \ 79*f08653c5Smvs ##__VA_ARGS__); \ 80*f08653c5Smvs if (mtx) \ 81*f08653c5Smvs mtx_leave(mtx); \ 82*f08653c5Smvs } \ 83*f08653c5Smvs } while (0) 8458360b13Sdlg 8558360b13Sdlg #define CONTAINER_OF(ptr, type, member) ({ \ 8658360b13Sdlg const __typeof( ((type *)0)->member ) *__mptr = (ptr); \ 8758360b13Sdlg (type *)( (char *)__mptr - offsetof(type,member) );}) 8858360b13Sdlg 8958360b13Sdlg /* First byte indicating packet type on the wire */ 9058360b13Sdlg #define WG_PKT_INITIATION htole32(1) 9158360b13Sdlg #define WG_PKT_RESPONSE htole32(2) 9258360b13Sdlg #define WG_PKT_COOKIE htole32(3) 9358360b13Sdlg #define WG_PKT_DATA htole32(4) 9458360b13Sdlg 9558360b13Sdlg #define WG_PKT_WITH_PADDING(n) (((n) + (16-1)) & (~(16-1))) 9658360b13Sdlg #define WG_KEY_SIZE WG_KEY_LEN 9758360b13Sdlg 9858360b13Sdlg struct wg_pkt_initiation { 9958360b13Sdlg uint32_t t; 10058360b13Sdlg uint32_t s_idx; 10158360b13Sdlg uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 10258360b13Sdlg uint8_t es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN]; 10358360b13Sdlg uint8_t ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN]; 10458360b13Sdlg struct cookie_macs m; 10558360b13Sdlg }; 10658360b13Sdlg 10758360b13Sdlg struct wg_pkt_response { 10858360b13Sdlg uint32_t t; 10958360b13Sdlg uint32_t s_idx; 11058360b13Sdlg uint32_t r_idx; 11158360b13Sdlg uint8_t ue[NOISE_PUBLIC_KEY_LEN]; 11258360b13Sdlg uint8_t en[0 + NOISE_AUTHTAG_LEN]; 11358360b13Sdlg struct cookie_macs m; 11458360b13Sdlg }; 11558360b13Sdlg 11658360b13Sdlg struct wg_pkt_cookie { 11758360b13Sdlg uint32_t t; 11858360b13Sdlg uint32_t r_idx; 11958360b13Sdlg uint8_t nonce[COOKIE_NONCE_SIZE]; 12058360b13Sdlg uint8_t ec[COOKIE_ENCRYPTED_SIZE]; 12158360b13Sdlg }; 12258360b13Sdlg 12358360b13Sdlg struct wg_pkt_data { 12458360b13Sdlg uint32_t t; 12558360b13Sdlg uint32_t r_idx; 12658360b13Sdlg uint8_t nonce[sizeof(uint64_t)]; 12758360b13Sdlg uint8_t buf[]; 12858360b13Sdlg }; 12958360b13Sdlg 13058360b13Sdlg struct wg_endpoint { 13158360b13Sdlg union { 13258360b13Sdlg struct sockaddr r_sa; 13358360b13Sdlg struct sockaddr_in r_sin; 13458360b13Sdlg #ifdef INET6 13558360b13Sdlg struct sockaddr_in6 r_sin6; 13658360b13Sdlg #endif 13758360b13Sdlg } e_remote; 13858360b13Sdlg union { 13958360b13Sdlg struct in_addr l_in; 14058360b13Sdlg #ifdef INET6 14158360b13Sdlg struct in6_pktinfo l_pktinfo6; 14258360b13Sdlg #define l_in6 l_pktinfo6.ipi6_addr 14358360b13Sdlg #endif 14458360b13Sdlg } e_local; 14558360b13Sdlg }; 14658360b13Sdlg 14758360b13Sdlg struct wg_tag { 14858360b13Sdlg struct wg_endpoint t_endpoint; 14958360b13Sdlg struct wg_peer *t_peer; 15058360b13Sdlg struct mbuf *t_mbuf; 15158360b13Sdlg int t_done; 15258360b13Sdlg int t_mtu; 15358360b13Sdlg }; 15458360b13Sdlg 15558360b13Sdlg struct wg_index { 15658360b13Sdlg LIST_ENTRY(wg_index) i_entry; 15758360b13Sdlg SLIST_ENTRY(wg_index) i_unused_entry; 15858360b13Sdlg uint32_t i_key; 15958360b13Sdlg struct noise_remote *i_value; 16058360b13Sdlg }; 16158360b13Sdlg 16258360b13Sdlg struct wg_timers { 163efbb2e09Smvs /* t_mtx is for blocking wg_timers_event_* when setting t_disabled. */ 164efbb2e09Smvs struct mutex t_mtx; 16558360b13Sdlg 16658360b13Sdlg int t_disabled; 16758360b13Sdlg int t_need_another_keepalive; 16858360b13Sdlg uint16_t t_persistent_keepalive_interval; 16958360b13Sdlg struct timeout t_new_handshake; 17058360b13Sdlg struct timeout t_send_keepalive; 17158360b13Sdlg struct timeout t_retry_handshake; 17258360b13Sdlg struct timeout t_zero_key_material; 17358360b13Sdlg struct timeout t_persistent_keepalive; 17458360b13Sdlg 17558360b13Sdlg struct mutex t_handshake_mtx; 17658360b13Sdlg struct timespec t_handshake_last_sent; /* nanouptime */ 17758360b13Sdlg struct timespec t_handshake_complete; /* nanotime */ 17858360b13Sdlg int t_handshake_retries; 17958360b13Sdlg }; 18058360b13Sdlg 18158360b13Sdlg struct wg_aip { 18258360b13Sdlg struct art_node a_node; 18358360b13Sdlg LIST_ENTRY(wg_aip) a_entry; 18458360b13Sdlg struct wg_peer *a_peer; 18558360b13Sdlg struct wg_aip_io a_data; 18658360b13Sdlg }; 18758360b13Sdlg 18858360b13Sdlg struct wg_queue { 18958360b13Sdlg struct mutex q_mtx; 19058360b13Sdlg struct mbuf_list q_list; 19158360b13Sdlg }; 19258360b13Sdlg 19358360b13Sdlg struct wg_ring { 19458360b13Sdlg struct mutex r_mtx; 19558360b13Sdlg uint32_t r_head; 19658360b13Sdlg uint32_t r_tail; 19758360b13Sdlg struct mbuf *r_buf[MAX_QUEUED_PKT]; 19858360b13Sdlg }; 19958360b13Sdlg 20058360b13Sdlg struct wg_peer { 20158360b13Sdlg LIST_ENTRY(wg_peer) p_pubkey_entry; 20258360b13Sdlg TAILQ_ENTRY(wg_peer) p_seq_entry; 20358360b13Sdlg uint64_t p_id; 20458360b13Sdlg struct wg_softc *p_sc; 20558360b13Sdlg 20658360b13Sdlg struct noise_remote p_remote; 20758360b13Sdlg struct cookie_maker p_cookie; 20858360b13Sdlg struct wg_timers p_timers; 20958360b13Sdlg 21058360b13Sdlg struct mutex p_counters_mtx; 21158360b13Sdlg uint64_t p_counters_tx; 21258360b13Sdlg uint64_t p_counters_rx; 21358360b13Sdlg 21458360b13Sdlg struct mutex p_endpoint_mtx; 21558360b13Sdlg struct wg_endpoint p_endpoint; 21658360b13Sdlg 21758360b13Sdlg struct task p_send_initiation; 21858360b13Sdlg struct task p_send_keepalive; 21958360b13Sdlg struct task p_clear_secrets; 22058360b13Sdlg struct task p_deliver_out; 22158360b13Sdlg struct task p_deliver_in; 22258360b13Sdlg 22358360b13Sdlg struct mbuf_queue p_stage_queue; 22458360b13Sdlg struct wg_queue p_encap_queue; 22558360b13Sdlg struct wg_queue p_decap_queue; 22658360b13Sdlg 22758360b13Sdlg SLIST_HEAD(,wg_index) p_unused_index; 22858360b13Sdlg struct wg_index p_index[3]; 22958360b13Sdlg 23058360b13Sdlg LIST_HEAD(,wg_aip) p_aip; 23158360b13Sdlg 23258360b13Sdlg SLIST_ENTRY(wg_peer) p_start_list; 23358360b13Sdlg int p_start_onlist; 23411d9cfa5Skn 23511d9cfa5Skn char p_description[IFDESCRSIZE]; 23658360b13Sdlg }; 23758360b13Sdlg 23858360b13Sdlg struct wg_softc { 23958360b13Sdlg struct ifnet sc_if; 24058360b13Sdlg SIPHASH_KEY sc_secret; 24158360b13Sdlg 24258360b13Sdlg struct rwlock sc_lock; 24358360b13Sdlg struct noise_local sc_local; 24458360b13Sdlg struct cookie_checker sc_cookie; 24558360b13Sdlg in_port_t sc_udp_port; 24658360b13Sdlg int sc_udp_rtable; 24758360b13Sdlg 24858360b13Sdlg struct rwlock sc_so_lock; 24958360b13Sdlg struct socket *sc_so4; 25058360b13Sdlg #ifdef INET6 25158360b13Sdlg struct socket *sc_so6; 25258360b13Sdlg #endif 25358360b13Sdlg 25458360b13Sdlg size_t sc_aip_num; 25558360b13Sdlg struct art_root *sc_aip4; 25658360b13Sdlg #ifdef INET6 25758360b13Sdlg struct art_root *sc_aip6; 25858360b13Sdlg #endif 25958360b13Sdlg 26058360b13Sdlg struct rwlock sc_peer_lock; 26158360b13Sdlg size_t sc_peer_num; 26258360b13Sdlg LIST_HEAD(,wg_peer) *sc_peer; 26358360b13Sdlg TAILQ_HEAD(,wg_peer) sc_peer_seq; 26458360b13Sdlg u_long sc_peer_mask; 26558360b13Sdlg 26658360b13Sdlg struct mutex sc_index_mtx; 26758360b13Sdlg LIST_HEAD(,wg_index) *sc_index; 26858360b13Sdlg u_long sc_index_mask; 26958360b13Sdlg 27058360b13Sdlg struct task sc_handshake; 27158360b13Sdlg struct mbuf_queue sc_handshake_queue; 27258360b13Sdlg 27358360b13Sdlg struct task sc_encap; 27458360b13Sdlg struct task sc_decap; 27558360b13Sdlg struct wg_ring sc_encap_ring; 27658360b13Sdlg struct wg_ring sc_decap_ring; 27758360b13Sdlg }; 27858360b13Sdlg 27958360b13Sdlg struct wg_peer * 28058360b13Sdlg wg_peer_create(struct wg_softc *, uint8_t[WG_KEY_SIZE]); 28158360b13Sdlg struct wg_peer * 28258360b13Sdlg wg_peer_lookup(struct wg_softc *, const uint8_t[WG_KEY_SIZE]); 28358360b13Sdlg void wg_peer_destroy(struct wg_peer *); 28458360b13Sdlg void wg_peer_set_endpoint_from_tag(struct wg_peer *, struct wg_tag *); 28558360b13Sdlg void wg_peer_set_sockaddr(struct wg_peer *, struct sockaddr *); 28658360b13Sdlg int wg_peer_get_sockaddr(struct wg_peer *, struct sockaddr *); 28758360b13Sdlg void wg_peer_clear_src(struct wg_peer *); 28858360b13Sdlg void wg_peer_get_endpoint(struct wg_peer *, struct wg_endpoint *); 28958360b13Sdlg void wg_peer_counters_add(struct wg_peer *, uint64_t, uint64_t); 29058360b13Sdlg 29158360b13Sdlg int wg_aip_add(struct wg_softc *, struct wg_peer *, struct wg_aip_io *); 29258360b13Sdlg struct wg_peer * 29358360b13Sdlg wg_aip_lookup(struct art_root *, void *); 29458360b13Sdlg int wg_aip_remove(struct wg_softc *, struct wg_peer *, 29558360b13Sdlg struct wg_aip_io *); 29658360b13Sdlg 29758360b13Sdlg int wg_socket_open(struct socket **, int, in_port_t *, int *, void *); 29858360b13Sdlg void wg_socket_close(struct socket **); 29958360b13Sdlg int wg_bind(struct wg_softc *, in_port_t *, int *); 30058360b13Sdlg void wg_unbind(struct wg_softc *); 30158360b13Sdlg int wg_send(struct wg_softc *, struct wg_endpoint *, struct mbuf *); 30258360b13Sdlg void wg_send_buf(struct wg_softc *, struct wg_endpoint *, uint8_t *, 30358360b13Sdlg size_t); 30458360b13Sdlg 30558360b13Sdlg struct wg_tag * 30658360b13Sdlg wg_tag_get(struct mbuf *); 30758360b13Sdlg 30858360b13Sdlg void wg_timers_init(struct wg_timers *); 30958360b13Sdlg void wg_timers_enable(struct wg_timers *); 31058360b13Sdlg void wg_timers_disable(struct wg_timers *); 31158360b13Sdlg void wg_timers_set_persistent_keepalive(struct wg_timers *, uint16_t); 31258360b13Sdlg int wg_timers_get_persistent_keepalive(struct wg_timers *, uint16_t *); 31358360b13Sdlg void wg_timers_get_last_handshake(struct wg_timers *, struct timespec *); 31458360b13Sdlg int wg_timers_expired_handshake_last_sent(struct wg_timers *); 31558360b13Sdlg int wg_timers_check_handshake_last_sent(struct wg_timers *); 31658360b13Sdlg 31758360b13Sdlg void wg_timers_event_data_sent(struct wg_timers *); 31858360b13Sdlg void wg_timers_event_data_received(struct wg_timers *); 31958360b13Sdlg void wg_timers_event_any_authenticated_packet_sent(struct wg_timers *); 32058360b13Sdlg void wg_timers_event_any_authenticated_packet_received(struct wg_timers *); 32158360b13Sdlg void wg_timers_event_handshake_initiated(struct wg_timers *); 32258360b13Sdlg void wg_timers_event_handshake_responded(struct wg_timers *); 32358360b13Sdlg void wg_timers_event_handshake_complete(struct wg_timers *); 32458360b13Sdlg void wg_timers_event_session_derived(struct wg_timers *); 32558360b13Sdlg void wg_timers_event_any_authenticated_packet_traversal(struct wg_timers *); 32658360b13Sdlg void wg_timers_event_want_initiation(struct wg_timers *); 32758360b13Sdlg void wg_timers_event_reset_handshake_last_sent(struct wg_timers *); 32858360b13Sdlg 32958360b13Sdlg void wg_timers_run_send_initiation(void *, int); 33058360b13Sdlg void wg_timers_run_retry_handshake(void *); 33158360b13Sdlg void wg_timers_run_send_keepalive(void *); 33258360b13Sdlg void wg_timers_run_new_handshake(void *); 33358360b13Sdlg void wg_timers_run_zero_key_material(void *); 33458360b13Sdlg void wg_timers_run_persistent_keepalive(void *); 33558360b13Sdlg 33658360b13Sdlg void wg_peer_send_buf(struct wg_peer *, uint8_t *, size_t); 33758360b13Sdlg void wg_send_initiation(void *); 33858360b13Sdlg void wg_send_response(struct wg_peer *); 33958360b13Sdlg void wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t, 34094dfe323Stb struct wg_endpoint *); 34158360b13Sdlg void wg_send_keepalive(void *); 34258360b13Sdlg void wg_peer_clear_secrets(void *); 34358360b13Sdlg void wg_handshake(struct wg_softc *, struct mbuf *); 34458360b13Sdlg void wg_handshake_worker(void *); 34558360b13Sdlg 34658360b13Sdlg void wg_encap(struct wg_softc *, struct mbuf *); 34758360b13Sdlg void wg_decap(struct wg_softc *, struct mbuf *); 34858360b13Sdlg void wg_encap_worker(void *); 34958360b13Sdlg void wg_decap_worker(void *); 35058360b13Sdlg void wg_deliver_out(void *); 35158360b13Sdlg void wg_deliver_in(void *); 35258360b13Sdlg 35358360b13Sdlg int wg_queue_in(struct wg_softc *, struct wg_peer *, struct mbuf *); 35458360b13Sdlg void wg_queue_out(struct wg_softc *, struct wg_peer *); 35558360b13Sdlg struct mbuf * 35658360b13Sdlg wg_ring_dequeue(struct wg_ring *); 35758360b13Sdlg struct mbuf * 35858360b13Sdlg wg_queue_dequeue(struct wg_queue *, struct wg_tag **); 35958360b13Sdlg size_t wg_queue_len(struct wg_queue *); 36058360b13Sdlg 36158360b13Sdlg struct noise_remote * 36258360b13Sdlg wg_remote_get(void *, uint8_t[NOISE_PUBLIC_KEY_LEN]); 36358360b13Sdlg uint32_t 36458360b13Sdlg wg_index_set(void *, struct noise_remote *); 36558360b13Sdlg struct noise_remote * 36658360b13Sdlg wg_index_get(void *, uint32_t); 36758360b13Sdlg void wg_index_drop(void *, uint32_t); 36858360b13Sdlg 36958360b13Sdlg struct mbuf * 37058360b13Sdlg wg_input(void *, struct mbuf *, struct ip *, struct ip6_hdr *, void *, 37158360b13Sdlg int); 37258360b13Sdlg int wg_output(struct ifnet *, struct mbuf *, struct sockaddr *, 37358360b13Sdlg struct rtentry *); 37458360b13Sdlg int wg_ioctl_set(struct wg_softc *, struct wg_data_io *); 37558360b13Sdlg int wg_ioctl_get(struct wg_softc *, struct wg_data_io *); 37658360b13Sdlg int wg_ioctl(struct ifnet *, u_long, caddr_t); 37758360b13Sdlg int wg_up(struct wg_softc *); 37858360b13Sdlg void wg_down(struct wg_softc *); 37958360b13Sdlg 38058360b13Sdlg int wg_clone_create(struct if_clone *, int); 38158360b13Sdlg int wg_clone_destroy(struct ifnet *); 38258360b13Sdlg void wgattach(int); 38358360b13Sdlg 38458360b13Sdlg uint64_t peer_counter = 0; 38558360b13Sdlg struct pool wg_aip_pool; 38658360b13Sdlg struct pool wg_peer_pool; 38758360b13Sdlg struct pool wg_ratelimit_pool; 38858360b13Sdlg struct timeval underload_interval = { UNDERLOAD_TIMEOUT, 0 }; 38958360b13Sdlg 39058360b13Sdlg size_t wg_counter = 0; 39158360b13Sdlg struct taskq *wg_handshake_taskq; 39258360b13Sdlg struct taskq *wg_crypt_taskq; 39358360b13Sdlg 39458360b13Sdlg struct if_clone wg_cloner = 39558360b13Sdlg IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 39658360b13Sdlg 39758360b13Sdlg struct wg_peer * 39858360b13Sdlg wg_peer_create(struct wg_softc *sc, uint8_t public[WG_KEY_SIZE]) 39958360b13Sdlg { 40058360b13Sdlg struct wg_peer *peer; 40158360b13Sdlg uint64_t idx; 40258360b13Sdlg 40358360b13Sdlg rw_assert_wrlock(&sc->sc_lock); 40458360b13Sdlg 40558360b13Sdlg if (sc->sc_peer_num >= MAX_PEERS_PER_IFACE) 40658360b13Sdlg return NULL; 40758360b13Sdlg 40858360b13Sdlg if ((peer = pool_get(&wg_peer_pool, PR_NOWAIT)) == NULL) 40958360b13Sdlg return NULL; 41058360b13Sdlg 41158360b13Sdlg peer->p_id = peer_counter++; 41258360b13Sdlg peer->p_sc = sc; 41358360b13Sdlg 41458360b13Sdlg noise_remote_init(&peer->p_remote, public, &sc->sc_local); 41558360b13Sdlg cookie_maker_init(&peer->p_cookie, public); 41658360b13Sdlg wg_timers_init(&peer->p_timers); 41758360b13Sdlg 41858360b13Sdlg mtx_init(&peer->p_counters_mtx, IPL_NET); 41958360b13Sdlg peer->p_counters_tx = 0; 42058360b13Sdlg peer->p_counters_rx = 0; 42158360b13Sdlg 42211d9cfa5Skn strlcpy(peer->p_description, "", IFDESCRSIZE); 42311d9cfa5Skn 42458360b13Sdlg mtx_init(&peer->p_endpoint_mtx, IPL_NET); 42558360b13Sdlg bzero(&peer->p_endpoint, sizeof(peer->p_endpoint)); 42658360b13Sdlg 42758360b13Sdlg task_set(&peer->p_send_initiation, wg_send_initiation, peer); 42858360b13Sdlg task_set(&peer->p_send_keepalive, wg_send_keepalive, peer); 42958360b13Sdlg task_set(&peer->p_clear_secrets, wg_peer_clear_secrets, peer); 43058360b13Sdlg task_set(&peer->p_deliver_out, wg_deliver_out, peer); 43158360b13Sdlg task_set(&peer->p_deliver_in, wg_deliver_in, peer); 43258360b13Sdlg 43358360b13Sdlg mq_init(&peer->p_stage_queue, MAX_STAGED_PKT, IPL_NET); 43458360b13Sdlg mtx_init(&peer->p_encap_queue.q_mtx, IPL_NET); 43558360b13Sdlg ml_init(&peer->p_encap_queue.q_list); 43658360b13Sdlg mtx_init(&peer->p_decap_queue.q_mtx, IPL_NET); 43758360b13Sdlg ml_init(&peer->p_decap_queue.q_list); 43858360b13Sdlg 43958360b13Sdlg SLIST_INIT(&peer->p_unused_index); 44058360b13Sdlg SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[0], 44158360b13Sdlg i_unused_entry); 44258360b13Sdlg SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[1], 44358360b13Sdlg i_unused_entry); 44458360b13Sdlg SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[2], 44558360b13Sdlg i_unused_entry); 44658360b13Sdlg 44758360b13Sdlg LIST_INIT(&peer->p_aip); 44858360b13Sdlg 44958360b13Sdlg peer->p_start_onlist = 0; 45058360b13Sdlg 45158360b13Sdlg idx = SipHash24(&sc->sc_secret, public, WG_KEY_SIZE); 45258360b13Sdlg idx &= sc->sc_peer_mask; 45358360b13Sdlg 45458360b13Sdlg rw_enter_write(&sc->sc_peer_lock); 45558360b13Sdlg LIST_INSERT_HEAD(&sc->sc_peer[idx], peer, p_pubkey_entry); 45658360b13Sdlg TAILQ_INSERT_TAIL(&sc->sc_peer_seq, peer, p_seq_entry); 45758360b13Sdlg sc->sc_peer_num++; 45858360b13Sdlg rw_exit_write(&sc->sc_peer_lock); 45958360b13Sdlg 460*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Peer %llu created\n", peer->p_id); 46158360b13Sdlg return peer; 46258360b13Sdlg } 46358360b13Sdlg 46458360b13Sdlg struct wg_peer * 46558360b13Sdlg wg_peer_lookup(struct wg_softc *sc, const uint8_t public[WG_KEY_SIZE]) 46658360b13Sdlg { 46758360b13Sdlg uint8_t peer_key[WG_KEY_SIZE]; 46858360b13Sdlg struct wg_peer *peer; 46958360b13Sdlg uint64_t idx; 47058360b13Sdlg 47158360b13Sdlg idx = SipHash24(&sc->sc_secret, public, WG_KEY_SIZE); 47258360b13Sdlg idx &= sc->sc_peer_mask; 47358360b13Sdlg 47458360b13Sdlg rw_enter_read(&sc->sc_peer_lock); 47558360b13Sdlg LIST_FOREACH(peer, &sc->sc_peer[idx], p_pubkey_entry) { 47658360b13Sdlg noise_remote_keys(&peer->p_remote, peer_key, NULL); 47758360b13Sdlg if (timingsafe_bcmp(peer_key, public, WG_KEY_SIZE) == 0) 47858360b13Sdlg goto done; 47958360b13Sdlg } 48058360b13Sdlg peer = NULL; 48158360b13Sdlg done: 48258360b13Sdlg rw_exit_read(&sc->sc_peer_lock); 48358360b13Sdlg return peer; 48458360b13Sdlg } 48558360b13Sdlg 48658360b13Sdlg void 48758360b13Sdlg wg_peer_destroy(struct wg_peer *peer) 48858360b13Sdlg { 48958360b13Sdlg struct wg_softc *sc = peer->p_sc; 49058360b13Sdlg struct wg_aip *aip, *taip; 49158360b13Sdlg 49258360b13Sdlg rw_assert_wrlock(&sc->sc_lock); 49358360b13Sdlg 49458360b13Sdlg /* 49558360b13Sdlg * Remove peer from the pubkey hashtable and disable all timeouts. 49658360b13Sdlg * After this, and flushing wg_handshake_taskq, then no more handshakes 49758360b13Sdlg * can be started. 49858360b13Sdlg */ 49958360b13Sdlg rw_enter_write(&sc->sc_peer_lock); 50058360b13Sdlg LIST_REMOVE(peer, p_pubkey_entry); 50158360b13Sdlg TAILQ_REMOVE(&sc->sc_peer_seq, peer, p_seq_entry); 50258360b13Sdlg sc->sc_peer_num--; 50358360b13Sdlg rw_exit_write(&sc->sc_peer_lock); 50458360b13Sdlg 50558360b13Sdlg wg_timers_disable(&peer->p_timers); 50658360b13Sdlg 50758360b13Sdlg taskq_barrier(wg_handshake_taskq); 50858360b13Sdlg 50958360b13Sdlg /* 51058360b13Sdlg * Now we drop all allowed ips, to drop all outgoing packets to the 51158360b13Sdlg * peer. Then drop all the indexes to drop all incoming packets to the 51258360b13Sdlg * peer. Then we can flush if_snd, wg_crypt_taskq and then nettq to 51358360b13Sdlg * ensure no more references to the peer exist. 51458360b13Sdlg */ 51558360b13Sdlg LIST_FOREACH_SAFE(aip, &peer->p_aip, a_entry, taip) 51658360b13Sdlg wg_aip_remove(sc, peer, &aip->a_data); 51758360b13Sdlg 51858360b13Sdlg noise_remote_clear(&peer->p_remote); 51958360b13Sdlg 52058360b13Sdlg NET_LOCK(); 52158360b13Sdlg while (!ifq_empty(&sc->sc_if.if_snd)) { 5228c2c9fccSmvs /* 5238c2c9fccSmvs * XXX: `if_snd' of stopped interface could still 5248c2c9fccSmvs * contain packets 5258c2c9fccSmvs */ 5268c2c9fccSmvs if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)) { 5278c2c9fccSmvs ifq_purge(&sc->sc_if.if_snd); 5288c2c9fccSmvs continue; 5298c2c9fccSmvs } 53058360b13Sdlg NET_UNLOCK(); 5314bd216efSmvs tsleep_nsec(&nowake, PWAIT, "wg_ifq", 1000); 53258360b13Sdlg NET_LOCK(); 53358360b13Sdlg } 53458360b13Sdlg NET_UNLOCK(); 53558360b13Sdlg 53658360b13Sdlg taskq_barrier(wg_crypt_taskq); 53758360b13Sdlg taskq_barrier(net_tq(sc->sc_if.if_index)); 53858360b13Sdlg 53976ac1d3dSyasuoka if (!mq_empty(&peer->p_stage_queue)) 54076ac1d3dSyasuoka mq_purge(&peer->p_stage_queue); 54176ac1d3dSyasuoka 542*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Peer %llu destroyed\n", peer->p_id); 54358360b13Sdlg explicit_bzero(peer, sizeof(*peer)); 54458360b13Sdlg pool_put(&wg_peer_pool, peer); 54558360b13Sdlg } 54658360b13Sdlg 54758360b13Sdlg void 54858360b13Sdlg wg_peer_set_endpoint_from_tag(struct wg_peer *peer, struct wg_tag *t) 54958360b13Sdlg { 55058360b13Sdlg if (memcmp(&t->t_endpoint, &peer->p_endpoint, 55158360b13Sdlg sizeof(t->t_endpoint)) == 0) 55258360b13Sdlg return; 55358360b13Sdlg 55458360b13Sdlg mtx_enter(&peer->p_endpoint_mtx); 55558360b13Sdlg peer->p_endpoint = t->t_endpoint; 55658360b13Sdlg mtx_leave(&peer->p_endpoint_mtx); 55758360b13Sdlg } 55858360b13Sdlg 55958360b13Sdlg void 56058360b13Sdlg wg_peer_set_sockaddr(struct wg_peer *peer, struct sockaddr *remote) 56158360b13Sdlg { 56258360b13Sdlg mtx_enter(&peer->p_endpoint_mtx); 56358360b13Sdlg memcpy(&peer->p_endpoint.e_remote, remote, 56458360b13Sdlg sizeof(peer->p_endpoint.e_remote)); 56558360b13Sdlg bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local)); 56658360b13Sdlg mtx_leave(&peer->p_endpoint_mtx); 56758360b13Sdlg } 56858360b13Sdlg 56958360b13Sdlg int 57058360b13Sdlg wg_peer_get_sockaddr(struct wg_peer *peer, struct sockaddr *remote) 57158360b13Sdlg { 57258360b13Sdlg int ret = 0; 57358360b13Sdlg 57458360b13Sdlg mtx_enter(&peer->p_endpoint_mtx); 57558360b13Sdlg if (peer->p_endpoint.e_remote.r_sa.sa_family != AF_UNSPEC) 57658360b13Sdlg memcpy(remote, &peer->p_endpoint.e_remote, 57758360b13Sdlg sizeof(peer->p_endpoint.e_remote)); 57858360b13Sdlg else 57958360b13Sdlg ret = ENOENT; 58058360b13Sdlg mtx_leave(&peer->p_endpoint_mtx); 58158360b13Sdlg return ret; 58258360b13Sdlg } 58358360b13Sdlg 58458360b13Sdlg void 58558360b13Sdlg wg_peer_clear_src(struct wg_peer *peer) 58658360b13Sdlg { 58758360b13Sdlg mtx_enter(&peer->p_endpoint_mtx); 58858360b13Sdlg bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local)); 58958360b13Sdlg mtx_leave(&peer->p_endpoint_mtx); 59058360b13Sdlg } 59158360b13Sdlg 59258360b13Sdlg void 59358360b13Sdlg wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *endpoint) 59458360b13Sdlg { 59558360b13Sdlg mtx_enter(&peer->p_endpoint_mtx); 59658360b13Sdlg memcpy(endpoint, &peer->p_endpoint, sizeof(*endpoint)); 59758360b13Sdlg mtx_leave(&peer->p_endpoint_mtx); 59858360b13Sdlg } 59958360b13Sdlg 60058360b13Sdlg void 60158360b13Sdlg wg_peer_counters_add(struct wg_peer *peer, uint64_t tx, uint64_t rx) 60258360b13Sdlg { 60358360b13Sdlg mtx_enter(&peer->p_counters_mtx); 60458360b13Sdlg peer->p_counters_tx += tx; 60558360b13Sdlg peer->p_counters_rx += rx; 60658360b13Sdlg mtx_leave(&peer->p_counters_mtx); 60758360b13Sdlg } 60858360b13Sdlg 60958360b13Sdlg int 61058360b13Sdlg wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, struct wg_aip_io *d) 61158360b13Sdlg { 61258360b13Sdlg struct art_root *root; 61358360b13Sdlg struct art_node *node; 61458360b13Sdlg struct wg_aip *aip; 61558360b13Sdlg int ret = 0; 61658360b13Sdlg 61758360b13Sdlg switch (d->a_af) { 61858360b13Sdlg case AF_INET: root = sc->sc_aip4; break; 61958360b13Sdlg #ifdef INET6 62058360b13Sdlg case AF_INET6: root = sc->sc_aip6; break; 62158360b13Sdlg #endif 62258360b13Sdlg default: return EAFNOSUPPORT; 62358360b13Sdlg } 62458360b13Sdlg 6254023ddd7Sderaadt if ((aip = pool_get(&wg_aip_pool, PR_NOWAIT|PR_ZERO)) == NULL) 62658360b13Sdlg return ENOBUFS; 62758360b13Sdlg 62858360b13Sdlg rw_enter_write(&root->ar_lock); 62958360b13Sdlg node = art_insert(root, &aip->a_node, &d->a_addr, d->a_cidr); 63058360b13Sdlg 63158360b13Sdlg if (node == &aip->a_node) { 63258360b13Sdlg aip->a_peer = peer; 63358360b13Sdlg aip->a_data = *d; 63458360b13Sdlg LIST_INSERT_HEAD(&peer->p_aip, aip, a_entry); 63558360b13Sdlg sc->sc_aip_num++; 63658360b13Sdlg } else { 63758360b13Sdlg pool_put(&wg_aip_pool, aip); 63858360b13Sdlg aip = (struct wg_aip *) node; 63958360b13Sdlg if (aip->a_peer != peer) { 64058360b13Sdlg LIST_REMOVE(aip, a_entry); 64158360b13Sdlg LIST_INSERT_HEAD(&peer->p_aip, aip, a_entry); 64258360b13Sdlg aip->a_peer = peer; 64358360b13Sdlg } 64458360b13Sdlg } 64558360b13Sdlg rw_exit_write(&root->ar_lock); 64658360b13Sdlg return ret; 64758360b13Sdlg } 64858360b13Sdlg 64958360b13Sdlg struct wg_peer * 65058360b13Sdlg wg_aip_lookup(struct art_root *root, void *addr) 65158360b13Sdlg { 65258360b13Sdlg struct srp_ref sr; 65358360b13Sdlg struct art_node *node; 65458360b13Sdlg 65558360b13Sdlg node = art_match(root, addr, &sr); 65658360b13Sdlg srp_leave(&sr); 65758360b13Sdlg 65858360b13Sdlg return node == NULL ? NULL : ((struct wg_aip *) node)->a_peer; 65958360b13Sdlg } 66058360b13Sdlg 66158360b13Sdlg int 66258360b13Sdlg wg_aip_remove(struct wg_softc *sc, struct wg_peer *peer, struct wg_aip_io *d) 66358360b13Sdlg { 66458360b13Sdlg struct srp_ref sr; 66558360b13Sdlg struct art_root *root; 66658360b13Sdlg struct art_node *node; 66758360b13Sdlg struct wg_aip *aip; 66858360b13Sdlg int ret = 0; 66958360b13Sdlg 67058360b13Sdlg switch (d->a_af) { 67158360b13Sdlg case AF_INET: root = sc->sc_aip4; break; 67258360b13Sdlg #ifdef INET6 67358360b13Sdlg case AF_INET6: root = sc->sc_aip6; break; 67458360b13Sdlg #endif 67558360b13Sdlg default: return EAFNOSUPPORT; 67658360b13Sdlg } 67758360b13Sdlg 67858360b13Sdlg rw_enter_write(&root->ar_lock); 67958360b13Sdlg if ((node = art_lookup(root, &d->a_addr, d->a_cidr, &sr)) == NULL) { 68058360b13Sdlg ret = ENOENT; 68158360b13Sdlg } else if (((struct wg_aip *) node)->a_peer != peer) { 68258360b13Sdlg ret = EXDEV; 68358360b13Sdlg } else { 68458360b13Sdlg aip = (struct wg_aip *)node; 68558360b13Sdlg if (art_delete(root, node, &d->a_addr, d->a_cidr) == NULL) 68658360b13Sdlg panic("art_delete failed to delete node %p", node); 68758360b13Sdlg 68858360b13Sdlg sc->sc_aip_num--; 68958360b13Sdlg LIST_REMOVE(aip, a_entry); 69058360b13Sdlg pool_put(&wg_aip_pool, aip); 69158360b13Sdlg } 69258360b13Sdlg 69358360b13Sdlg srp_leave(&sr); 69458360b13Sdlg rw_exit_write(&root->ar_lock); 69558360b13Sdlg return ret; 69658360b13Sdlg } 69758360b13Sdlg 69858360b13Sdlg int 69958360b13Sdlg wg_socket_open(struct socket **so, int af, in_port_t *port, 70058360b13Sdlg int *rtable, void *upcall_arg) 70158360b13Sdlg { 70258360b13Sdlg struct mbuf mhostnam, mrtable; 70358360b13Sdlg #ifdef INET6 70458360b13Sdlg struct sockaddr_in6 *sin6; 70558360b13Sdlg #endif 70658360b13Sdlg struct sockaddr_in *sin; 70784245c07Sclaudio int ret; 70858360b13Sdlg 70958360b13Sdlg m_inithdr(&mhostnam); 71058360b13Sdlg m_inithdr(&mrtable); 71158360b13Sdlg 71258360b13Sdlg bzero(mtod(&mrtable, u_int *), sizeof(u_int)); 71358360b13Sdlg *mtod(&mrtable, u_int *) = *rtable; 71458360b13Sdlg mrtable.m_len = sizeof(u_int); 71558360b13Sdlg 71658360b13Sdlg if (af == AF_INET) { 71758360b13Sdlg sin = mtod(&mhostnam, struct sockaddr_in *); 71858360b13Sdlg bzero(sin, sizeof(*sin)); 71958360b13Sdlg sin->sin_len = sizeof(*sin); 72058360b13Sdlg sin->sin_family = AF_INET; 72158360b13Sdlg sin->sin_port = *port; 72258360b13Sdlg sin->sin_addr.s_addr = INADDR_ANY; 72358360b13Sdlg mhostnam.m_len = sin->sin_len; 72458360b13Sdlg #ifdef INET6 72558360b13Sdlg } else if (af == AF_INET6) { 72658360b13Sdlg sin6 = mtod(&mhostnam, struct sockaddr_in6 *); 72758360b13Sdlg bzero(sin6, sizeof(*sin6)); 72858360b13Sdlg sin6->sin6_len = sizeof(*sin6); 72958360b13Sdlg sin6->sin6_family = AF_INET6; 73058360b13Sdlg sin6->sin6_port = *port; 73158360b13Sdlg sin6->sin6_addr = (struct in6_addr) { .s6_addr = { 0 } }; 73258360b13Sdlg mhostnam.m_len = sin6->sin6_len; 73358360b13Sdlg #endif 73458360b13Sdlg } else { 73558360b13Sdlg return EAFNOSUPPORT; 73658360b13Sdlg } 73758360b13Sdlg 73858360b13Sdlg if ((ret = socreate(af, so, SOCK_DGRAM, 0)) != 0) 73958360b13Sdlg return ret; 74058360b13Sdlg 74184245c07Sclaudio solock(*so); 74258360b13Sdlg sotoinpcb(*so)->inp_upcall = wg_input; 74358360b13Sdlg sotoinpcb(*so)->inp_upcall_arg = upcall_arg; 744ee078639Smvs sounlock(*so); 74558360b13Sdlg 74658360b13Sdlg if ((ret = sosetopt(*so, SOL_SOCKET, SO_RTABLE, &mrtable)) == 0) { 747ee078639Smvs solock(*so); 74858360b13Sdlg if ((ret = sobind(*so, &mhostnam, curproc)) == 0) { 74958360b13Sdlg *port = sotoinpcb(*so)->inp_lport; 75058360b13Sdlg *rtable = sotoinpcb(*so)->inp_rtableid; 75158360b13Sdlg } 75284245c07Sclaudio sounlock(*so); 753ee078639Smvs } 75458360b13Sdlg 75558360b13Sdlg if (ret != 0) 75658360b13Sdlg wg_socket_close(so); 75758360b13Sdlg 75858360b13Sdlg return ret; 75958360b13Sdlg } 76058360b13Sdlg 76158360b13Sdlg void 76258360b13Sdlg wg_socket_close(struct socket **so) 76358360b13Sdlg { 76458360b13Sdlg if (*so != NULL && soclose(*so, 0) != 0) 76558360b13Sdlg panic("Unable to close wg socket"); 76658360b13Sdlg *so = NULL; 76758360b13Sdlg } 76858360b13Sdlg 76958360b13Sdlg int 77058360b13Sdlg wg_bind(struct wg_softc *sc, in_port_t *portp, int *rtablep) 77158360b13Sdlg { 77258360b13Sdlg int ret = 0, rtable = *rtablep; 77358360b13Sdlg in_port_t port = *portp; 77458360b13Sdlg struct socket *so4; 77558360b13Sdlg #ifdef INET6 77658360b13Sdlg struct socket *so6; 77758360b13Sdlg int retries = 0; 77858360b13Sdlg retry: 77958360b13Sdlg #endif 78058360b13Sdlg if ((ret = wg_socket_open(&so4, AF_INET, &port, &rtable, sc)) != 0) 78158360b13Sdlg return ret; 78258360b13Sdlg 78358360b13Sdlg #ifdef INET6 78458360b13Sdlg if ((ret = wg_socket_open(&so6, AF_INET6, &port, &rtable, sc)) != 0) { 78558360b13Sdlg if (ret == EADDRINUSE && *portp == 0 && retries++ < 100) 78658360b13Sdlg goto retry; 78758360b13Sdlg wg_socket_close(&so4); 78858360b13Sdlg return ret; 78958360b13Sdlg } 79058360b13Sdlg #endif 79158360b13Sdlg 79258360b13Sdlg rw_enter_write(&sc->sc_so_lock); 79358360b13Sdlg wg_socket_close(&sc->sc_so4); 79458360b13Sdlg sc->sc_so4 = so4; 79558360b13Sdlg #ifdef INET6 79658360b13Sdlg wg_socket_close(&sc->sc_so6); 79758360b13Sdlg sc->sc_so6 = so6; 79858360b13Sdlg #endif 79958360b13Sdlg rw_exit_write(&sc->sc_so_lock); 80058360b13Sdlg 80158360b13Sdlg *portp = port; 80258360b13Sdlg *rtablep = rtable; 80358360b13Sdlg return 0; 80458360b13Sdlg } 80558360b13Sdlg 80658360b13Sdlg void 80758360b13Sdlg wg_unbind(struct wg_softc *sc) 80858360b13Sdlg { 80958360b13Sdlg rw_enter_write(&sc->sc_so_lock); 81058360b13Sdlg wg_socket_close(&sc->sc_so4); 81158360b13Sdlg #ifdef INET6 81258360b13Sdlg wg_socket_close(&sc->sc_so6); 81358360b13Sdlg #endif 81458360b13Sdlg rw_exit_write(&sc->sc_so_lock); 81558360b13Sdlg } 81658360b13Sdlg 81758360b13Sdlg int 81858360b13Sdlg wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m) 81958360b13Sdlg { 82058360b13Sdlg struct mbuf peernam, *control = NULL; 82158360b13Sdlg int ret; 82258360b13Sdlg 82358360b13Sdlg /* Get local control address before locking */ 82458360b13Sdlg if (e->e_remote.r_sa.sa_family == AF_INET) { 82558360b13Sdlg if (e->e_local.l_in.s_addr != INADDR_ANY) 82658360b13Sdlg control = sbcreatecontrol(&e->e_local.l_in, 82758360b13Sdlg sizeof(struct in_addr), IP_SENDSRCADDR, 82858360b13Sdlg IPPROTO_IP); 82958360b13Sdlg #ifdef INET6 83058360b13Sdlg } else if (e->e_remote.r_sa.sa_family == AF_INET6) { 83158360b13Sdlg if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6)) 83258360b13Sdlg control = sbcreatecontrol(&e->e_local.l_pktinfo6, 83358360b13Sdlg sizeof(struct in6_pktinfo), IPV6_PKTINFO, 83458360b13Sdlg IPPROTO_IPV6); 83558360b13Sdlg #endif 83658360b13Sdlg } else { 8372f84ac13Ssthen m_freem(m); 83858360b13Sdlg return EAFNOSUPPORT; 83958360b13Sdlg } 84058360b13Sdlg 84158360b13Sdlg /* Get remote address */ 84258360b13Sdlg peernam.m_type = MT_SONAME; 84358360b13Sdlg peernam.m_next = NULL; 84458360b13Sdlg peernam.m_nextpkt = NULL; 84558360b13Sdlg peernam.m_data = (void *)&e->e_remote.r_sa; 84658360b13Sdlg peernam.m_len = e->e_remote.r_sa.sa_len; 84758360b13Sdlg peernam.m_flags = 0; 84858360b13Sdlg 84958360b13Sdlg rw_enter_read(&sc->sc_so_lock); 85058360b13Sdlg if (e->e_remote.r_sa.sa_family == AF_INET && sc->sc_so4 != NULL) 85158360b13Sdlg ret = sosend(sc->sc_so4, &peernam, NULL, m, control, 0); 85258360b13Sdlg #ifdef INET6 85358360b13Sdlg else if (e->e_remote.r_sa.sa_family == AF_INET6 && sc->sc_so6 != NULL) 85458360b13Sdlg ret = sosend(sc->sc_so6, &peernam, NULL, m, control, 0); 85558360b13Sdlg #endif 85658360b13Sdlg else { 85758360b13Sdlg ret = ENOTCONN; 85858360b13Sdlg m_freem(control); 85958360b13Sdlg m_freem(m); 86058360b13Sdlg } 86158360b13Sdlg rw_exit_read(&sc->sc_so_lock); 86258360b13Sdlg 86358360b13Sdlg return ret; 86458360b13Sdlg } 86558360b13Sdlg 86658360b13Sdlg void 86758360b13Sdlg wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, uint8_t *buf, 86858360b13Sdlg size_t len) 86958360b13Sdlg { 87058360b13Sdlg struct mbuf *m; 87158360b13Sdlg int ret = 0; 872c0619985Sclaudio size_t mlen = len + max_hdr; 87358360b13Sdlg 87458360b13Sdlg retry: 87558360b13Sdlg m = m_gethdr(M_WAIT, MT_DATA); 876c0619985Sclaudio if (mlen > MHLEN) 877c0619985Sclaudio MCLGETL(m, M_WAIT, mlen); 878c0619985Sclaudio m_align(m, len); 879c0619985Sclaudio m->m_pkthdr.len = m->m_len = len; 880c0619985Sclaudio memcpy(mtod(m, void *), buf, len); 88158360b13Sdlg 88258360b13Sdlg /* As we're sending a handshake packet here, we want high priority */ 88358360b13Sdlg m->m_pkthdr.pf.prio = IFQ_MAXPRIO; 88458360b13Sdlg 88558360b13Sdlg if (ret == 0) { 88658360b13Sdlg ret = wg_send(sc, e, m); 88758360b13Sdlg /* Retry if we couldn't bind to e->e_local */ 88858360b13Sdlg if (ret == EADDRNOTAVAIL) { 88958360b13Sdlg bzero(&e->e_local, sizeof(e->e_local)); 89058360b13Sdlg goto retry; 89158360b13Sdlg } 89258360b13Sdlg } else { 89358360b13Sdlg ret = wg_send(sc, e, m); 89458360b13Sdlg if (ret != 0) 895*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, 896*f08653c5Smvs "Unable to send packet\n"); 89758360b13Sdlg } 89858360b13Sdlg } 89958360b13Sdlg 90058360b13Sdlg struct wg_tag * 90158360b13Sdlg wg_tag_get(struct mbuf *m) 90258360b13Sdlg { 90358360b13Sdlg struct m_tag *mtag; 90458360b13Sdlg 90558360b13Sdlg if ((mtag = m_tag_find(m, PACKET_TAG_WIREGUARD, NULL)) == NULL) { 90658360b13Sdlg mtag = m_tag_get(PACKET_TAG_WIREGUARD, sizeof(struct wg_tag), 90758360b13Sdlg M_NOWAIT); 90858360b13Sdlg if (mtag == NULL) 90958360b13Sdlg return (NULL); 91058360b13Sdlg bzero(mtag + 1, sizeof(struct wg_tag)); 91158360b13Sdlg m_tag_prepend(m, mtag); 91258360b13Sdlg } 91358360b13Sdlg return ((struct wg_tag *)(mtag + 1)); 91458360b13Sdlg } 91558360b13Sdlg 91658360b13Sdlg /* 91758360b13Sdlg * The following section handles the timeout callbacks for a WireGuard session. 918678831beSjsg * These functions provide an "event based" model for controlling wg(8) session 91958360b13Sdlg * timers. All function calls occur after the specified event below. 92058360b13Sdlg * 92158360b13Sdlg * wg_timers_event_data_sent: 92258360b13Sdlg * tx: data 92358360b13Sdlg * wg_timers_event_data_received: 92458360b13Sdlg * rx: data 92558360b13Sdlg * wg_timers_event_any_authenticated_packet_sent: 92658360b13Sdlg * tx: keepalive, data, handshake 92758360b13Sdlg * wg_timers_event_any_authenticated_packet_received: 92858360b13Sdlg * rx: keepalive, data, handshake 92958360b13Sdlg * wg_timers_event_any_authenticated_packet_traversal: 93058360b13Sdlg * tx, rx: keepalive, data, handshake 93158360b13Sdlg * wg_timers_event_handshake_initiated: 93258360b13Sdlg * tx: initiation 93358360b13Sdlg * wg_timers_event_handshake_responded: 93458360b13Sdlg * tx: response 93558360b13Sdlg * wg_timers_event_handshake_complete: 93658360b13Sdlg * rx: response, confirmation data 93758360b13Sdlg * wg_timers_event_session_derived: 93858360b13Sdlg * tx: response, rx: response 93958360b13Sdlg * wg_timers_event_want_initiation: 94058360b13Sdlg * tx: data failed, old keys expiring 94158360b13Sdlg * wg_timers_event_reset_handshake_last_sent: 94258360b13Sdlg * anytime we may immediately want a new handshake 94358360b13Sdlg */ 94458360b13Sdlg void 94558360b13Sdlg wg_timers_init(struct wg_timers *t) 94658360b13Sdlg { 94758360b13Sdlg bzero(t, sizeof(*t)); 948efbb2e09Smvs mtx_init_flags(&t->t_mtx, IPL_NET, "wg_timers", 0); 94958360b13Sdlg mtx_init(&t->t_handshake_mtx, IPL_NET); 95058360b13Sdlg 95158360b13Sdlg timeout_set(&t->t_new_handshake, wg_timers_run_new_handshake, t); 95258360b13Sdlg timeout_set(&t->t_send_keepalive, wg_timers_run_send_keepalive, t); 95358360b13Sdlg timeout_set(&t->t_retry_handshake, wg_timers_run_retry_handshake, t); 95458360b13Sdlg timeout_set(&t->t_persistent_keepalive, 95558360b13Sdlg wg_timers_run_persistent_keepalive, t); 95658360b13Sdlg timeout_set(&t->t_zero_key_material, 95758360b13Sdlg wg_timers_run_zero_key_material, t); 95858360b13Sdlg } 95958360b13Sdlg 96058360b13Sdlg void 96158360b13Sdlg wg_timers_enable(struct wg_timers *t) 96258360b13Sdlg { 963efbb2e09Smvs mtx_enter(&t->t_mtx); 96458360b13Sdlg t->t_disabled = 0; 965efbb2e09Smvs mtx_leave(&t->t_mtx); 96658360b13Sdlg wg_timers_run_persistent_keepalive(t); 96758360b13Sdlg } 96858360b13Sdlg 96958360b13Sdlg void 97058360b13Sdlg wg_timers_disable(struct wg_timers *t) 97158360b13Sdlg { 972efbb2e09Smvs mtx_enter(&t->t_mtx); 97358360b13Sdlg t->t_disabled = 1; 97458360b13Sdlg t->t_need_another_keepalive = 0; 975efbb2e09Smvs mtx_leave(&t->t_mtx); 97658360b13Sdlg 97758360b13Sdlg timeout_del_barrier(&t->t_new_handshake); 97858360b13Sdlg timeout_del_barrier(&t->t_send_keepalive); 97958360b13Sdlg timeout_del_barrier(&t->t_retry_handshake); 98058360b13Sdlg timeout_del_barrier(&t->t_persistent_keepalive); 98158360b13Sdlg timeout_del_barrier(&t->t_zero_key_material); 98258360b13Sdlg } 98358360b13Sdlg 98458360b13Sdlg void 98558360b13Sdlg wg_timers_set_persistent_keepalive(struct wg_timers *t, uint16_t interval) 98658360b13Sdlg { 987efbb2e09Smvs mtx_enter(&t->t_mtx); 98858360b13Sdlg if (!t->t_disabled) { 98958360b13Sdlg t->t_persistent_keepalive_interval = interval; 99058360b13Sdlg wg_timers_run_persistent_keepalive(t); 99158360b13Sdlg } 992efbb2e09Smvs mtx_leave(&t->t_mtx); 99358360b13Sdlg } 99458360b13Sdlg 99558360b13Sdlg int 99658360b13Sdlg wg_timers_get_persistent_keepalive(struct wg_timers *t, uint16_t *interval) 99758360b13Sdlg { 99858360b13Sdlg *interval = t->t_persistent_keepalive_interval; 99958360b13Sdlg return *interval > 0 ? 0 : ENOENT; 100058360b13Sdlg } 100158360b13Sdlg 100258360b13Sdlg void 100358360b13Sdlg wg_timers_get_last_handshake(struct wg_timers *t, struct timespec *time) 100458360b13Sdlg { 100558360b13Sdlg mtx_enter(&t->t_handshake_mtx); 100658360b13Sdlg *time = t->t_handshake_complete; 100758360b13Sdlg mtx_leave(&t->t_handshake_mtx); 100858360b13Sdlg } 100958360b13Sdlg 101058360b13Sdlg int 101158360b13Sdlg wg_timers_expired_handshake_last_sent(struct wg_timers *t) 101258360b13Sdlg { 101358360b13Sdlg struct timespec uptime; 101458360b13Sdlg struct timespec expire = { .tv_sec = REKEY_TIMEOUT, .tv_nsec = 0 }; 101558360b13Sdlg 101658360b13Sdlg getnanouptime(&uptime); 101758360b13Sdlg timespecadd(&t->t_handshake_last_sent, &expire, &expire); 101858360b13Sdlg return timespeccmp(&uptime, &expire, >) ? ETIMEDOUT : 0; 101958360b13Sdlg } 102058360b13Sdlg 102158360b13Sdlg int 102258360b13Sdlg wg_timers_check_handshake_last_sent(struct wg_timers *t) 102358360b13Sdlg { 102458360b13Sdlg int ret; 102558360b13Sdlg mtx_enter(&t->t_handshake_mtx); 102658360b13Sdlg if ((ret = wg_timers_expired_handshake_last_sent(t)) == ETIMEDOUT) 102758360b13Sdlg getnanouptime(&t->t_handshake_last_sent); 102858360b13Sdlg mtx_leave(&t->t_handshake_mtx); 102958360b13Sdlg return ret; 103058360b13Sdlg } 103158360b13Sdlg 103258360b13Sdlg void 103358360b13Sdlg wg_timers_event_data_sent(struct wg_timers *t) 103458360b13Sdlg { 103558360b13Sdlg int msecs = NEW_HANDSHAKE_TIMEOUT * 1000; 103658360b13Sdlg msecs += arc4random_uniform(REKEY_TIMEOUT_JITTER); 103758360b13Sdlg 1038efbb2e09Smvs mtx_enter(&t->t_mtx); 103958360b13Sdlg if (!t->t_disabled && !timeout_pending(&t->t_new_handshake)) 104058360b13Sdlg timeout_add_msec(&t->t_new_handshake, msecs); 1041efbb2e09Smvs mtx_leave(&t->t_mtx); 104258360b13Sdlg } 104358360b13Sdlg 104458360b13Sdlg void 104558360b13Sdlg wg_timers_event_data_received(struct wg_timers *t) 104658360b13Sdlg { 1047efbb2e09Smvs mtx_enter(&t->t_mtx); 104858360b13Sdlg if (!t->t_disabled) { 104958360b13Sdlg if (!timeout_pending(&t->t_send_keepalive)) 105058360b13Sdlg timeout_add_sec(&t->t_send_keepalive, 105158360b13Sdlg KEEPALIVE_TIMEOUT); 105258360b13Sdlg else 105358360b13Sdlg t->t_need_another_keepalive = 1; 105458360b13Sdlg } 1055efbb2e09Smvs mtx_leave(&t->t_mtx); 105658360b13Sdlg } 105758360b13Sdlg 105858360b13Sdlg void 105958360b13Sdlg wg_timers_event_any_authenticated_packet_sent(struct wg_timers *t) 106058360b13Sdlg { 106158360b13Sdlg timeout_del(&t->t_send_keepalive); 106258360b13Sdlg } 106358360b13Sdlg 106458360b13Sdlg void 106558360b13Sdlg wg_timers_event_any_authenticated_packet_received(struct wg_timers *t) 106658360b13Sdlg { 106758360b13Sdlg timeout_del(&t->t_new_handshake); 106858360b13Sdlg } 106958360b13Sdlg 107058360b13Sdlg void 107158360b13Sdlg wg_timers_event_any_authenticated_packet_traversal(struct wg_timers *t) 107258360b13Sdlg { 1073efbb2e09Smvs mtx_enter(&t->t_mtx); 107458360b13Sdlg if (!t->t_disabled && t->t_persistent_keepalive_interval > 0) 107558360b13Sdlg timeout_add_sec(&t->t_persistent_keepalive, 107658360b13Sdlg t->t_persistent_keepalive_interval); 1077efbb2e09Smvs mtx_leave(&t->t_mtx); 107858360b13Sdlg } 107958360b13Sdlg 108058360b13Sdlg void 108158360b13Sdlg wg_timers_event_handshake_initiated(struct wg_timers *t) 108258360b13Sdlg { 108358360b13Sdlg int msecs = REKEY_TIMEOUT * 1000; 108458360b13Sdlg msecs += arc4random_uniform(REKEY_TIMEOUT_JITTER); 108558360b13Sdlg 1086efbb2e09Smvs mtx_enter(&t->t_mtx); 108758360b13Sdlg if (!t->t_disabled) 108858360b13Sdlg timeout_add_msec(&t->t_retry_handshake, msecs); 1089efbb2e09Smvs mtx_leave(&t->t_mtx); 109058360b13Sdlg } 109158360b13Sdlg 109258360b13Sdlg void 109358360b13Sdlg wg_timers_event_handshake_responded(struct wg_timers *t) 109458360b13Sdlg { 109558360b13Sdlg mtx_enter(&t->t_handshake_mtx); 109658360b13Sdlg getnanouptime(&t->t_handshake_last_sent); 109758360b13Sdlg mtx_leave(&t->t_handshake_mtx); 109858360b13Sdlg } 109958360b13Sdlg 110058360b13Sdlg void 110158360b13Sdlg wg_timers_event_handshake_complete(struct wg_timers *t) 110258360b13Sdlg { 1103efbb2e09Smvs mtx_enter(&t->t_mtx); 110458360b13Sdlg if (!t->t_disabled) { 110558360b13Sdlg mtx_enter(&t->t_handshake_mtx); 110658360b13Sdlg timeout_del(&t->t_retry_handshake); 110758360b13Sdlg t->t_handshake_retries = 0; 110858360b13Sdlg getnanotime(&t->t_handshake_complete); 110958360b13Sdlg mtx_leave(&t->t_handshake_mtx); 111058360b13Sdlg wg_timers_run_send_keepalive(t); 111158360b13Sdlg } 1112efbb2e09Smvs mtx_leave(&t->t_mtx); 111358360b13Sdlg } 111458360b13Sdlg 111558360b13Sdlg void 111658360b13Sdlg wg_timers_event_session_derived(struct wg_timers *t) 111758360b13Sdlg { 1118efbb2e09Smvs mtx_enter(&t->t_mtx); 111958360b13Sdlg if (!t->t_disabled) 112058360b13Sdlg timeout_add_sec(&t->t_zero_key_material, REJECT_AFTER_TIME * 3); 1121efbb2e09Smvs mtx_leave(&t->t_mtx); 112258360b13Sdlg } 112358360b13Sdlg 112458360b13Sdlg void 112558360b13Sdlg wg_timers_event_want_initiation(struct wg_timers *t) 112658360b13Sdlg { 1127efbb2e09Smvs mtx_enter(&t->t_mtx); 112858360b13Sdlg if (!t->t_disabled) 112958360b13Sdlg wg_timers_run_send_initiation(t, 0); 1130efbb2e09Smvs mtx_leave(&t->t_mtx); 113158360b13Sdlg } 113258360b13Sdlg 113358360b13Sdlg void 113458360b13Sdlg wg_timers_event_reset_handshake_last_sent(struct wg_timers *t) 113558360b13Sdlg { 113658360b13Sdlg mtx_enter(&t->t_handshake_mtx); 113758360b13Sdlg t->t_handshake_last_sent.tv_sec -= (REKEY_TIMEOUT + 1); 113858360b13Sdlg mtx_leave(&t->t_handshake_mtx); 113958360b13Sdlg } 114058360b13Sdlg 114158360b13Sdlg void 114258360b13Sdlg wg_timers_run_send_initiation(void *_t, int is_retry) 114358360b13Sdlg { 114458360b13Sdlg struct wg_timers *t = _t; 114558360b13Sdlg struct wg_peer *peer = CONTAINER_OF(t, struct wg_peer, p_timers); 114658360b13Sdlg if (!is_retry) 114758360b13Sdlg t->t_handshake_retries = 0; 114858360b13Sdlg if (wg_timers_expired_handshake_last_sent(t) == ETIMEDOUT) 114958360b13Sdlg task_add(wg_handshake_taskq, &peer->p_send_initiation); 115058360b13Sdlg } 115158360b13Sdlg 115258360b13Sdlg void 115358360b13Sdlg wg_timers_run_retry_handshake(void *_t) 115458360b13Sdlg { 115558360b13Sdlg struct wg_timers *t = _t; 115658360b13Sdlg struct wg_peer *peer = CONTAINER_OF(t, struct wg_peer, p_timers); 1157*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 115858360b13Sdlg 115958360b13Sdlg mtx_enter(&t->t_handshake_mtx); 116058360b13Sdlg if (t->t_handshake_retries <= MAX_TIMER_HANDSHAKES) { 116158360b13Sdlg t->t_handshake_retries++; 116258360b13Sdlg mtx_leave(&t->t_handshake_mtx); 116358360b13Sdlg 1164*f08653c5Smvs WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, 1165*f08653c5Smvs "Handshake for peer %llu (%s) did not complete after %d " 1166*f08653c5Smvs "seconds, retrying (try %d)\n", peer->p_id, 1167*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1168*f08653c5Smvs sizeof(ipaddr)), 116958360b13Sdlg REKEY_TIMEOUT, t->t_handshake_retries + 1); 117058360b13Sdlg wg_peer_clear_src(peer); 117158360b13Sdlg wg_timers_run_send_initiation(t, 1); 117258360b13Sdlg } else { 117358360b13Sdlg mtx_leave(&t->t_handshake_mtx); 117458360b13Sdlg 1175*f08653c5Smvs WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, 1176*f08653c5Smvs "Handshake for peer %llu (%s) did not complete after %d " 1177*f08653c5Smvs "retries, giving up\n", peer->p_id, 1178*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1179*f08653c5Smvs sizeof(ipaddr)), MAX_TIMER_HANDSHAKES + 2); 118058360b13Sdlg 118158360b13Sdlg timeout_del(&t->t_send_keepalive); 118258360b13Sdlg mq_purge(&peer->p_stage_queue); 118358360b13Sdlg if (!timeout_pending(&t->t_zero_key_material)) 118458360b13Sdlg timeout_add_sec(&t->t_zero_key_material, 118558360b13Sdlg REJECT_AFTER_TIME * 3); 118658360b13Sdlg } 118758360b13Sdlg } 118858360b13Sdlg 118958360b13Sdlg void 119058360b13Sdlg wg_timers_run_send_keepalive(void *_t) 119158360b13Sdlg { 119258360b13Sdlg struct wg_timers *t = _t; 119358360b13Sdlg struct wg_peer *peer = CONTAINER_OF(t, struct wg_peer, p_timers); 119458360b13Sdlg 119558360b13Sdlg task_add(wg_crypt_taskq, &peer->p_send_keepalive); 119658360b13Sdlg if (t->t_need_another_keepalive) { 119758360b13Sdlg t->t_need_another_keepalive = 0; 119858360b13Sdlg timeout_add_sec(&t->t_send_keepalive, KEEPALIVE_TIMEOUT); 119958360b13Sdlg } 120058360b13Sdlg } 120158360b13Sdlg 120258360b13Sdlg void 120358360b13Sdlg wg_timers_run_new_handshake(void *_t) 120458360b13Sdlg { 120558360b13Sdlg struct wg_timers *t = _t; 120658360b13Sdlg struct wg_peer *peer = CONTAINER_OF(t, struct wg_peer, p_timers); 1207*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 120858360b13Sdlg 1209*f08653c5Smvs WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, 1210*f08653c5Smvs "Retrying handshake with peer %llu (%s) because we " 1211*f08653c5Smvs "stopped hearing back after %d seconds\n", peer->p_id, 1212*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1213*f08653c5Smvs sizeof(ipaddr)), NEW_HANDSHAKE_TIMEOUT); 121458360b13Sdlg wg_peer_clear_src(peer); 121558360b13Sdlg 121658360b13Sdlg wg_timers_run_send_initiation(t, 0); 121758360b13Sdlg } 121858360b13Sdlg 121958360b13Sdlg void 122058360b13Sdlg wg_timers_run_zero_key_material(void *_t) 122158360b13Sdlg { 122258360b13Sdlg struct wg_timers *t = _t; 122358360b13Sdlg struct wg_peer *peer = CONTAINER_OF(t, struct wg_peer, p_timers); 1224*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 122558360b13Sdlg 1226*f08653c5Smvs WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, "Zeroing out " 1227*f08653c5Smvs "keys for peer %llu (%s)\n", peer->p_id, 1228*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1229*f08653c5Smvs sizeof(ipaddr))); 123058360b13Sdlg task_add(wg_handshake_taskq, &peer->p_clear_secrets); 123158360b13Sdlg } 123258360b13Sdlg 123358360b13Sdlg void 123458360b13Sdlg wg_timers_run_persistent_keepalive(void *_t) 123558360b13Sdlg { 123658360b13Sdlg struct wg_timers *t = _t; 123758360b13Sdlg struct wg_peer *peer = CONTAINER_OF(t, struct wg_peer, p_timers); 123858360b13Sdlg if (t->t_persistent_keepalive_interval != 0) 123958360b13Sdlg task_add(wg_crypt_taskq, &peer->p_send_keepalive); 124058360b13Sdlg } 124158360b13Sdlg 124258360b13Sdlg /* The following functions handle handshakes */ 124358360b13Sdlg void 124458360b13Sdlg wg_peer_send_buf(struct wg_peer *peer, uint8_t *buf, size_t len) 124558360b13Sdlg { 124658360b13Sdlg struct wg_endpoint endpoint; 124758360b13Sdlg 124858360b13Sdlg wg_peer_counters_add(peer, len, 0); 124958360b13Sdlg wg_timers_event_any_authenticated_packet_traversal(&peer->p_timers); 125058360b13Sdlg wg_timers_event_any_authenticated_packet_sent(&peer->p_timers); 125158360b13Sdlg wg_peer_get_endpoint(peer, &endpoint); 125258360b13Sdlg wg_send_buf(peer->p_sc, &endpoint, buf, len); 125358360b13Sdlg } 125458360b13Sdlg 125558360b13Sdlg void 125658360b13Sdlg wg_send_initiation(void *_peer) 125758360b13Sdlg { 125858360b13Sdlg struct wg_peer *peer = _peer; 125958360b13Sdlg struct wg_pkt_initiation pkt; 1260*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 126158360b13Sdlg 126258360b13Sdlg if (wg_timers_check_handshake_last_sent(&peer->p_timers) != ETIMEDOUT) 126358360b13Sdlg return; 126458360b13Sdlg 1265*f08653c5Smvs WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, "Sending " 1266*f08653c5Smvs "handshake initiation to peer %llu (%s)\n", peer->p_id, 1267*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1268*f08653c5Smvs sizeof(ipaddr))); 126958360b13Sdlg 127058360b13Sdlg if (noise_create_initiation(&peer->p_remote, &pkt.s_idx, pkt.ue, pkt.es, 127158360b13Sdlg pkt.ets) != 0) 127258360b13Sdlg return; 127358360b13Sdlg pkt.t = WG_PKT_INITIATION; 127458360b13Sdlg cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, 127558360b13Sdlg sizeof(pkt)-sizeof(pkt.m)); 127658360b13Sdlg wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt)); 127758360b13Sdlg wg_timers_event_handshake_initiated(&peer->p_timers); 127858360b13Sdlg } 127958360b13Sdlg 128058360b13Sdlg void 128158360b13Sdlg wg_send_response(struct wg_peer *peer) 128258360b13Sdlg { 128358360b13Sdlg struct wg_pkt_response pkt; 1284*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 128558360b13Sdlg 1286*f08653c5Smvs WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, "Sending " 1287*f08653c5Smvs "handshake response to peer %llu (%s)\n", peer->p_id, 1288*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1289*f08653c5Smvs sizeof(ipaddr))); 129058360b13Sdlg 129158360b13Sdlg if (noise_create_response(&peer->p_remote, &pkt.s_idx, &pkt.r_idx, 129258360b13Sdlg pkt.ue, pkt.en) != 0) 129358360b13Sdlg return; 129458360b13Sdlg if (noise_remote_begin_session(&peer->p_remote) != 0) 129558360b13Sdlg return; 129658360b13Sdlg wg_timers_event_session_derived(&peer->p_timers); 129758360b13Sdlg pkt.t = WG_PKT_RESPONSE; 129858360b13Sdlg cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt, 129958360b13Sdlg sizeof(pkt)-sizeof(pkt.m)); 130058360b13Sdlg wg_timers_event_handshake_responded(&peer->p_timers); 130158360b13Sdlg wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt)); 130258360b13Sdlg } 130358360b13Sdlg 130458360b13Sdlg void 130558360b13Sdlg wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx, 130658360b13Sdlg struct wg_endpoint *e) 130758360b13Sdlg { 130858360b13Sdlg struct wg_pkt_cookie pkt; 130958360b13Sdlg 1310*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Sending cookie response for denied " 1311*f08653c5Smvs "handshake message\n"); 131258360b13Sdlg 131358360b13Sdlg pkt.t = WG_PKT_COOKIE; 131458360b13Sdlg pkt.r_idx = idx; 131558360b13Sdlg 131658360b13Sdlg cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce, 131758360b13Sdlg pkt.ec, &e->e_remote.r_sa); 131858360b13Sdlg 131958360b13Sdlg wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt)); 132058360b13Sdlg } 132158360b13Sdlg 132258360b13Sdlg void 132358360b13Sdlg wg_send_keepalive(void *_peer) 132458360b13Sdlg { 132558360b13Sdlg struct wg_peer *peer = _peer; 132658360b13Sdlg struct wg_softc *sc = peer->p_sc; 132758360b13Sdlg struct wg_tag *t; 132858360b13Sdlg struct mbuf *m; 132958360b13Sdlg 133058360b13Sdlg if (!mq_empty(&peer->p_stage_queue)) 133158360b13Sdlg goto send; 133258360b13Sdlg 133358360b13Sdlg if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) 133458360b13Sdlg return; 133558360b13Sdlg 133658360b13Sdlg if ((t = wg_tag_get(m)) == NULL) { 133758360b13Sdlg m_freem(m); 133858360b13Sdlg return; 133958360b13Sdlg } 134058360b13Sdlg 134158360b13Sdlg t->t_peer = peer; 134258360b13Sdlg t->t_mbuf = NULL; 134358360b13Sdlg t->t_done = 0; 134458360b13Sdlg t->t_mtu = 0; /* MTU == 0 OK for keepalive */ 134558360b13Sdlg 134658360b13Sdlg mq_push(&peer->p_stage_queue, m); 134758360b13Sdlg send: 134858360b13Sdlg if (noise_remote_ready(&peer->p_remote) == 0) { 134958360b13Sdlg wg_queue_out(sc, peer); 135058360b13Sdlg task_add(wg_crypt_taskq, &sc->sc_encap); 135158360b13Sdlg } else { 135258360b13Sdlg wg_timers_event_want_initiation(&peer->p_timers); 135358360b13Sdlg } 135458360b13Sdlg } 135558360b13Sdlg 135658360b13Sdlg void 135758360b13Sdlg wg_peer_clear_secrets(void *_peer) 135858360b13Sdlg { 135958360b13Sdlg struct wg_peer *peer = _peer; 136058360b13Sdlg noise_remote_clear(&peer->p_remote); 136158360b13Sdlg } 136258360b13Sdlg 136358360b13Sdlg void 136458360b13Sdlg wg_handshake(struct wg_softc *sc, struct mbuf *m) 136558360b13Sdlg { 136658360b13Sdlg struct wg_tag *t; 136758360b13Sdlg struct wg_pkt_initiation *init; 136858360b13Sdlg struct wg_pkt_response *resp; 136958360b13Sdlg struct wg_pkt_cookie *cook; 137058360b13Sdlg struct wg_peer *peer; 137158360b13Sdlg struct noise_remote *remote; 137258360b13Sdlg int res, underload = 0; 137358360b13Sdlg static struct timeval wg_last_underload; /* microuptime */ 1374*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 137558360b13Sdlg 137658360b13Sdlg if (mq_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES/8) { 137758360b13Sdlg getmicrouptime(&wg_last_underload); 137858360b13Sdlg underload = 1; 137958360b13Sdlg } else if (wg_last_underload.tv_sec != 0) { 138058360b13Sdlg if (!ratecheck(&wg_last_underload, &underload_interval)) 138158360b13Sdlg underload = 1; 138258360b13Sdlg else 138358360b13Sdlg bzero(&wg_last_underload, sizeof(wg_last_underload)); 138458360b13Sdlg } 138558360b13Sdlg 138658360b13Sdlg t = wg_tag_get(m); 138758360b13Sdlg 138858360b13Sdlg switch (*mtod(m, uint32_t *)) { 138958360b13Sdlg case WG_PKT_INITIATION: 139058360b13Sdlg init = mtod(m, struct wg_pkt_initiation *); 139158360b13Sdlg 139258360b13Sdlg res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m, 139358360b13Sdlg init, sizeof(*init) - sizeof(init->m), 139458360b13Sdlg underload, &t->t_endpoint.e_remote.r_sa); 139558360b13Sdlg 139658360b13Sdlg if (res == EINVAL) { 1397*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Invalid initiation " 1398*f08653c5Smvs "MAC from %s\n", 1399*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1400*f08653c5Smvs sizeof(ipaddr))); 140158360b13Sdlg goto error; 140258360b13Sdlg } else if (res == ECONNREFUSED) { 1403*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Handshake " 1404*f08653c5Smvs "ratelimited from %s\n", 1405*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1406*f08653c5Smvs sizeof(ipaddr))); 140758360b13Sdlg goto error; 140858360b13Sdlg } else if (res == EAGAIN) { 140958360b13Sdlg wg_send_cookie(sc, &init->m, init->s_idx, 141058360b13Sdlg &t->t_endpoint); 141158360b13Sdlg goto error; 141258360b13Sdlg } else if (res != 0) { 14134123b6a7Sderaadt panic("unexpected response: %d", res); 141458360b13Sdlg } 141558360b13Sdlg 141658360b13Sdlg if (noise_consume_initiation(&sc->sc_local, &remote, 141758360b13Sdlg init->s_idx, init->ue, init->es, init->ets) != 0) { 1418*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Invalid handshake " 1419*f08653c5Smvs "initiation from %s\n", 1420*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1421*f08653c5Smvs sizeof(ipaddr))); 142258360b13Sdlg goto error; 142358360b13Sdlg } 142458360b13Sdlg 142558360b13Sdlg peer = CONTAINER_OF(remote, struct wg_peer, p_remote); 142658360b13Sdlg 1427*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Receiving handshake initiation " 1428*f08653c5Smvs "from peer %llu (%s)\n", peer->p_id, 1429*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1430*f08653c5Smvs sizeof(ipaddr))); 143158360b13Sdlg 143258360b13Sdlg wg_peer_counters_add(peer, 0, sizeof(*init)); 143358360b13Sdlg wg_peer_set_endpoint_from_tag(peer, t); 143458360b13Sdlg wg_send_response(peer); 143558360b13Sdlg break; 143658360b13Sdlg case WG_PKT_RESPONSE: 143758360b13Sdlg resp = mtod(m, struct wg_pkt_response *); 143858360b13Sdlg 143958360b13Sdlg res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m, 144058360b13Sdlg resp, sizeof(*resp) - sizeof(resp->m), 144158360b13Sdlg underload, &t->t_endpoint.e_remote.r_sa); 144258360b13Sdlg 144358360b13Sdlg if (res == EINVAL) { 1444*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Invalid response " 1445*f08653c5Smvs "MAC from %s\n", 1446*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1447*f08653c5Smvs sizeof(ipaddr))); 144858360b13Sdlg goto error; 144958360b13Sdlg } else if (res == ECONNREFUSED) { 1450*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Handshake " 1451*f08653c5Smvs "ratelimited from %s\n", 1452*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1453*f08653c5Smvs sizeof(ipaddr))); 145458360b13Sdlg goto error; 145558360b13Sdlg } else if (res == EAGAIN) { 145658360b13Sdlg wg_send_cookie(sc, &resp->m, resp->s_idx, 145758360b13Sdlg &t->t_endpoint); 145858360b13Sdlg goto error; 145958360b13Sdlg } else if (res != 0) { 14604123b6a7Sderaadt panic("unexpected response: %d", res); 146158360b13Sdlg } 146258360b13Sdlg 146358360b13Sdlg if ((remote = wg_index_get(sc, resp->r_idx)) == NULL) { 1464*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Unknown " 1465*f08653c5Smvs "handshake response from %s\n", 1466*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1467*f08653c5Smvs sizeof(ipaddr))); 146858360b13Sdlg goto error; 146958360b13Sdlg } 147058360b13Sdlg 147158360b13Sdlg peer = CONTAINER_OF(remote, struct wg_peer, p_remote); 147258360b13Sdlg 147358360b13Sdlg if (noise_consume_response(remote, resp->s_idx, resp->r_idx, 147458360b13Sdlg resp->ue, resp->en) != 0) { 1475*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Invalid handshake " 1476*f08653c5Smvs "response from %s\n", 1477*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1478*f08653c5Smvs sizeof(ipaddr))); 147958360b13Sdlg goto error; 148058360b13Sdlg } 148158360b13Sdlg 1482*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Receiving handshake response " 1483*f08653c5Smvs "from peer %llu (%s)\n", peer->p_id, 1484*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1485*f08653c5Smvs sizeof(ipaddr))); 148658360b13Sdlg 148758360b13Sdlg wg_peer_counters_add(peer, 0, sizeof(*resp)); 148858360b13Sdlg wg_peer_set_endpoint_from_tag(peer, t); 148958360b13Sdlg if (noise_remote_begin_session(&peer->p_remote) == 0) { 149058360b13Sdlg wg_timers_event_session_derived(&peer->p_timers); 149158360b13Sdlg wg_timers_event_handshake_complete(&peer->p_timers); 149258360b13Sdlg } 149358360b13Sdlg break; 149458360b13Sdlg case WG_PKT_COOKIE: 149558360b13Sdlg cook = mtod(m, struct wg_pkt_cookie *); 149658360b13Sdlg 149758360b13Sdlg if ((remote = wg_index_get(sc, cook->r_idx)) == NULL) { 1498*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Unknown cookie " 1499*f08653c5Smvs "index from %s\n", 1500*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1501*f08653c5Smvs sizeof(ipaddr))); 150258360b13Sdlg goto error; 150358360b13Sdlg } 150458360b13Sdlg 150558360b13Sdlg peer = CONTAINER_OF(remote, struct wg_peer, p_remote); 150658360b13Sdlg 150758360b13Sdlg if (cookie_maker_consume_payload(&peer->p_cookie, 150858360b13Sdlg cook->nonce, cook->ec) != 0) { 1509*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Could not decrypt " 1510*f08653c5Smvs "cookie response from %s\n", 1511*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1512*f08653c5Smvs sizeof(ipaddr))); 151358360b13Sdlg goto error; 151458360b13Sdlg } 151558360b13Sdlg 1516*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Receiving cookie response " 1517*f08653c5Smvs "from %s\n", 1518*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 1519*f08653c5Smvs sizeof(ipaddr))); 152058360b13Sdlg goto error; 152158360b13Sdlg default: 152258360b13Sdlg panic("invalid packet in handshake queue"); 152358360b13Sdlg } 152458360b13Sdlg 152558360b13Sdlg wg_timers_event_any_authenticated_packet_received(&peer->p_timers); 152658360b13Sdlg wg_timers_event_any_authenticated_packet_traversal(&peer->p_timers); 152758360b13Sdlg error: 152858360b13Sdlg m_freem(m); 152958360b13Sdlg } 153058360b13Sdlg 153158360b13Sdlg void 153258360b13Sdlg wg_handshake_worker(void *_sc) 153358360b13Sdlg { 153458360b13Sdlg struct mbuf *m; 153558360b13Sdlg struct wg_softc *sc = _sc; 153658360b13Sdlg while ((m = mq_dequeue(&sc->sc_handshake_queue)) != NULL) 153758360b13Sdlg wg_handshake(sc, m); 153858360b13Sdlg } 153958360b13Sdlg 154058360b13Sdlg /* 154158360b13Sdlg * The following functions handle encapsulation (encryption) and 154258360b13Sdlg * decapsulation (decryption). The wg_{en,de}cap functions will run in the 154358360b13Sdlg * sc_crypt_taskq, while wg_deliver_{in,out} must be serialised and will run 154458360b13Sdlg * in nettq. 154558360b13Sdlg * 154658360b13Sdlg * The packets are tracked in two queues, a serial queue and a parallel queue. 154758360b13Sdlg * - The parallel queue is used to distribute the encryption across multiple 154858360b13Sdlg * threads. 154958360b13Sdlg * - The serial queue ensures that packets are not reordered and are 1550b076d4fbSjsg * delivered in sequence. 155158360b13Sdlg * The wg_tag attached to the packet contains two flags to help the two queues 155258360b13Sdlg * interact. 155358360b13Sdlg * - t_done: The parallel queue has finished with the packet, now the serial 155458360b13Sdlg * queue can do it's work. 155558360b13Sdlg * - t_mbuf: Used to store the *crypted packet. in the case of encryption, 155658360b13Sdlg * this is a newly allocated packet, and in the case of decryption, 155758360b13Sdlg * it is a pointer to the same packet, that has been decrypted and 155858360b13Sdlg * truncated. If t_mbuf is NULL, then *cryption failed and this 155958360b13Sdlg * packet should not be passed. 156058360b13Sdlg * wg_{en,de}cap work on the parallel queue, while wg_deliver_{in,out} work 156158360b13Sdlg * on the serial queue. 156258360b13Sdlg */ 156358360b13Sdlg void 156458360b13Sdlg wg_encap(struct wg_softc *sc, struct mbuf *m) 156558360b13Sdlg { 156658360b13Sdlg int res = 0; 156758360b13Sdlg struct wg_pkt_data *data; 156858360b13Sdlg struct wg_peer *peer; 156958360b13Sdlg struct wg_tag *t; 157058360b13Sdlg struct mbuf *mc; 157158360b13Sdlg size_t padding_len, plaintext_len, out_len; 157258360b13Sdlg uint64_t nonce; 1573*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 157458360b13Sdlg 157558360b13Sdlg t = wg_tag_get(m); 157658360b13Sdlg peer = t->t_peer; 157758360b13Sdlg 1578c0619985Sclaudio plaintext_len = WG_PKT_WITH_PADDING(m->m_pkthdr.len); 157958360b13Sdlg padding_len = plaintext_len - m->m_pkthdr.len; 1580c0619985Sclaudio out_len = sizeof(struct wg_pkt_data) + plaintext_len + 1581c0619985Sclaudio NOISE_AUTHTAG_LEN; 158258360b13Sdlg 158358360b13Sdlg /* 158458360b13Sdlg * For the time being we allocate a new packet with sufficient size to 158558360b13Sdlg * hold the encrypted data and headers. It would be difficult to 158658360b13Sdlg * overcome as p_encap_queue (mbuf_list) holds a reference to the mbuf. 158758360b13Sdlg * If we m_makespace or similar, we risk corrupting that list. 158858360b13Sdlg * Additionally, we only pass a buf and buf length to 158958360b13Sdlg * noise_remote_encrypt. Technically it would be possible to teach 159058360b13Sdlg * noise_remote_encrypt about mbufs, but we would need to sort out the 159158360b13Sdlg * p_encap_queue situation first. 159258360b13Sdlg */ 1593c0619985Sclaudio if ((mc = m_clget(NULL, M_NOWAIT, out_len + max_hdr)) == NULL) 159458360b13Sdlg goto error; 1595c0619985Sclaudio m_align(mc, out_len); 159658360b13Sdlg 159758360b13Sdlg data = mtod(mc, struct wg_pkt_data *); 159858360b13Sdlg m_copydata(m, 0, m->m_pkthdr.len, data->buf); 159958360b13Sdlg bzero(data->buf + m->m_pkthdr.len, padding_len); 160058360b13Sdlg data->t = WG_PKT_DATA; 160158360b13Sdlg 160258360b13Sdlg /* 160358360b13Sdlg * Copy the flow hash from the inner packet to the outer packet, so 160458360b13Sdlg * that fq_codel can property separate streams, rather than falling 160558360b13Sdlg * back to random buckets. 160658360b13Sdlg */ 160758360b13Sdlg mc->m_pkthdr.ph_flowid = m->m_pkthdr.ph_flowid; 160858360b13Sdlg 1609dbebf518Ssthen mc->m_pkthdr.pf.prio = m->m_pkthdr.pf.prio; 1610dbebf518Ssthen 161158360b13Sdlg res = noise_remote_encrypt(&peer->p_remote, &data->r_idx, &nonce, 161258360b13Sdlg data->buf, plaintext_len); 161358360b13Sdlg nonce = htole64(nonce); /* Wire format is little endian. */ 161458360b13Sdlg memcpy(data->nonce, &nonce, sizeof(data->nonce)); 161558360b13Sdlg 161658360b13Sdlg if (__predict_false(res == EINVAL)) { 161758360b13Sdlg m_freem(mc); 161858360b13Sdlg goto error; 161958360b13Sdlg } else if (__predict_false(res == ESTALE)) { 162058360b13Sdlg wg_timers_event_want_initiation(&peer->p_timers); 162158360b13Sdlg } else if (__predict_false(res != 0)) { 16224123b6a7Sderaadt panic("unexpected result: %d", res); 162358360b13Sdlg } 162458360b13Sdlg 162558360b13Sdlg /* A packet with length 0 is a keepalive packet */ 162658360b13Sdlg if (__predict_false(m->m_pkthdr.len == 0)) 1627*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, &peer->p_endpoint_mtx, "Sending " 1628*f08653c5Smvs "keepalive packet to peer %llu (%s)\n", peer->p_id, 1629*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr, 1630*f08653c5Smvs sizeof(ipaddr))); 163158360b13Sdlg 163258360b13Sdlg mc->m_pkthdr.ph_loopcnt = m->m_pkthdr.ph_loopcnt; 163358360b13Sdlg mc->m_flags &= ~(M_MCAST | M_BCAST); 1634c0619985Sclaudio mc->m_pkthdr.len = mc->m_len = out_len; 163558360b13Sdlg 163658360b13Sdlg /* 163758360b13Sdlg * We would count ifc_opackets, ifc_obytes of m here, except if_snd 163858360b13Sdlg * already does that for us, so no need to worry about it. 163958360b13Sdlg counters_pkt(sc->sc_if.if_counters, ifc_opackets, ifc_obytes, 164058360b13Sdlg m->m_pkthdr.len); 164158360b13Sdlg */ 164258360b13Sdlg wg_peer_counters_add(peer, mc->m_pkthdr.len, 0); 164358360b13Sdlg 164458360b13Sdlg t->t_mbuf = mc; 164558360b13Sdlg error: 164658360b13Sdlg t->t_done = 1; 164758360b13Sdlg task_add(net_tq(sc->sc_if.if_index), &peer->p_deliver_out); 164858360b13Sdlg } 164958360b13Sdlg 165058360b13Sdlg void 165158360b13Sdlg wg_decap(struct wg_softc *sc, struct mbuf *m) 165258360b13Sdlg { 165358360b13Sdlg int res, len; 165458360b13Sdlg struct ip *ip; 165558360b13Sdlg struct ip6_hdr *ip6; 165658360b13Sdlg struct wg_pkt_data *data; 165758360b13Sdlg struct wg_peer *peer, *allowed_peer; 165858360b13Sdlg struct wg_tag *t; 165958360b13Sdlg size_t payload_len; 166058360b13Sdlg uint64_t nonce; 1661*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 166258360b13Sdlg 166358360b13Sdlg t = wg_tag_get(m); 166458360b13Sdlg peer = t->t_peer; 166558360b13Sdlg 166658360b13Sdlg /* 166758360b13Sdlg * Likewise to wg_encap, we pass a buf and buf length to 166858360b13Sdlg * noise_remote_decrypt. Again, possible to teach it about mbufs 166958360b13Sdlg * but need to get over the p_decap_queue situation first. However, 167058360b13Sdlg * we do not need to allocate a new mbuf as the decrypted packet is 167158360b13Sdlg * strictly smaller than encrypted. We just set t_mbuf to m and 167258360b13Sdlg * wg_deliver_in knows how to deal with that. 167358360b13Sdlg */ 167458360b13Sdlg data = mtod(m, struct wg_pkt_data *); 167558360b13Sdlg payload_len = m->m_pkthdr.len - sizeof(struct wg_pkt_data); 167658360b13Sdlg memcpy(&nonce, data->nonce, sizeof(nonce)); 167758360b13Sdlg nonce = le64toh(nonce); /* Wire format is little endian. */ 167858360b13Sdlg res = noise_remote_decrypt(&peer->p_remote, data->r_idx, nonce, 167958360b13Sdlg data->buf, payload_len); 168058360b13Sdlg 168158360b13Sdlg if (__predict_false(res == EINVAL)) { 168258360b13Sdlg goto error; 168358360b13Sdlg } else if (__predict_false(res == ECONNRESET)) { 168458360b13Sdlg wg_timers_event_handshake_complete(&peer->p_timers); 168558360b13Sdlg } else if (__predict_false(res == ESTALE)) { 168658360b13Sdlg wg_timers_event_want_initiation(&peer->p_timers); 168758360b13Sdlg } else if (__predict_false(res != 0)) { 16884123b6a7Sderaadt panic("unexpected response: %d", res); 168958360b13Sdlg } 169058360b13Sdlg 169158360b13Sdlg wg_peer_set_endpoint_from_tag(peer, t); 169258360b13Sdlg 169358360b13Sdlg wg_peer_counters_add(peer, 0, m->m_pkthdr.len); 169458360b13Sdlg 169558360b13Sdlg m_adj(m, sizeof(struct wg_pkt_data)); 169658360b13Sdlg m_adj(m, -NOISE_AUTHTAG_LEN); 169758360b13Sdlg 169858360b13Sdlg counters_pkt(sc->sc_if.if_counters, ifc_ipackets, ifc_ibytes, 169958360b13Sdlg m->m_pkthdr.len); 170058360b13Sdlg 170158360b13Sdlg /* A packet with length 0 is a keepalive packet */ 170258360b13Sdlg if (__predict_false(m->m_pkthdr.len == 0)) { 1703*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, &peer->p_endpoint_mtx, "Receiving " 1704*f08653c5Smvs "keepalive packet from peer %llu (%s)\n", peer->p_id, 1705*f08653c5Smvs sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, 1706*f08653c5Smvs ipaddr, sizeof(ipaddr))); 170758360b13Sdlg goto done; 170858360b13Sdlg } 170958360b13Sdlg 171058360b13Sdlg /* 171158360b13Sdlg * We can let the network stack handle the intricate validation of the 171258360b13Sdlg * IP header, we just worry about the sizeof and the version, so we can 171358360b13Sdlg * read the source address in wg_aip_lookup. 171458360b13Sdlg * 1715df8d9afdSjsg * We also need to trim the packet, as it was likely padded before 171658360b13Sdlg * encryption. While we could drop it here, it will be more helpful to 171758360b13Sdlg * pass it to bpf_mtap and use the counters that people are expecting 171858360b13Sdlg * in ipv4_input and ipv6_input. We can rely on ipv4_input and 171958360b13Sdlg * ipv6_input to properly validate the headers. 172058360b13Sdlg */ 172158360b13Sdlg ip = mtod(m, struct ip *); 172258360b13Sdlg ip6 = mtod(m, struct ip6_hdr *); 172358360b13Sdlg 172458360b13Sdlg if (m->m_pkthdr.len >= sizeof(struct ip) && ip->ip_v == IPVERSION) { 172558360b13Sdlg m->m_pkthdr.ph_family = AF_INET; 172658360b13Sdlg 172758360b13Sdlg len = ntohs(ip->ip_len); 172858360b13Sdlg if (len >= sizeof(struct ip) && len < m->m_pkthdr.len) 172958360b13Sdlg m_adj(m, len - m->m_pkthdr.len); 173058360b13Sdlg 173158360b13Sdlg allowed_peer = wg_aip_lookup(sc->sc_aip4, &ip->ip_src); 173258360b13Sdlg #ifdef INET6 173358360b13Sdlg } else if (m->m_pkthdr.len >= sizeof(struct ip6_hdr) && 173458360b13Sdlg (ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) { 173558360b13Sdlg m->m_pkthdr.ph_family = AF_INET6; 173658360b13Sdlg 173758360b13Sdlg len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr); 173858360b13Sdlg if (len < m->m_pkthdr.len) 173958360b13Sdlg m_adj(m, len - m->m_pkthdr.len); 174058360b13Sdlg 174158360b13Sdlg allowed_peer = wg_aip_lookup(sc->sc_aip6, &ip6->ip6_src); 174258360b13Sdlg #endif 174358360b13Sdlg } else { 1744*f08653c5Smvs WGPRINTF(LOG_WARNING, sc, &peer->p_endpoint_mtx, "Packet " 1745*f08653c5Smvs "is neither IPv4 nor IPv6 from peer %llu (%s)\n", 1746*f08653c5Smvs peer->p_id, sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, 1747*f08653c5Smvs ipaddr, sizeof(ipaddr))); 174858360b13Sdlg goto error; 174958360b13Sdlg } 175058360b13Sdlg 175158360b13Sdlg if (__predict_false(peer != allowed_peer)) { 1752*f08653c5Smvs WGPRINTF(LOG_WARNING, sc, &peer->p_endpoint_mtx, "Packet " 1753*f08653c5Smvs "has unallowed source IP from peer %llu (%s)\n", 1754*f08653c5Smvs peer->p_id, sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, 1755*f08653c5Smvs ipaddr, sizeof(ipaddr))); 175658360b13Sdlg goto error; 175758360b13Sdlg } 175858360b13Sdlg 17598069430eSprocter /* tunneled packet was not offloaded */ 17608069430eSprocter m->m_pkthdr.csum_flags = 0; 176158360b13Sdlg 176258360b13Sdlg m->m_pkthdr.ph_ifidx = sc->sc_if.if_index; 176358360b13Sdlg m->m_pkthdr.ph_rtableid = sc->sc_if.if_rdomain; 176458360b13Sdlg m->m_flags &= ~(M_MCAST | M_BCAST); 176589fdc1a9Skn #if NPF > 0 176658360b13Sdlg pf_pkt_addr_changed(m); 176789fdc1a9Skn #endif /* NPF > 0 */ 176858360b13Sdlg 176958360b13Sdlg done: 177058360b13Sdlg t->t_mbuf = m; 177158360b13Sdlg error: 177258360b13Sdlg t->t_done = 1; 177358360b13Sdlg task_add(net_tq(sc->sc_if.if_index), &peer->p_deliver_in); 177458360b13Sdlg } 177558360b13Sdlg 177658360b13Sdlg void 177758360b13Sdlg wg_encap_worker(void *_sc) 177858360b13Sdlg { 177958360b13Sdlg struct mbuf *m; 178058360b13Sdlg struct wg_softc *sc = _sc; 178158360b13Sdlg while ((m = wg_ring_dequeue(&sc->sc_encap_ring)) != NULL) 178258360b13Sdlg wg_encap(sc, m); 178358360b13Sdlg } 178458360b13Sdlg 178558360b13Sdlg void 178658360b13Sdlg wg_decap_worker(void *_sc) 178758360b13Sdlg { 178858360b13Sdlg struct mbuf *m; 178958360b13Sdlg struct wg_softc *sc = _sc; 179058360b13Sdlg while ((m = wg_ring_dequeue(&sc->sc_decap_ring)) != NULL) 179158360b13Sdlg wg_decap(sc, m); 179258360b13Sdlg } 179358360b13Sdlg 179458360b13Sdlg void 179558360b13Sdlg wg_deliver_out(void *_peer) 179658360b13Sdlg { 179758360b13Sdlg struct wg_peer *peer = _peer; 179858360b13Sdlg struct wg_softc *sc = peer->p_sc; 179958360b13Sdlg struct wg_endpoint endpoint; 180058360b13Sdlg struct wg_tag *t; 180158360b13Sdlg struct mbuf *m; 180258360b13Sdlg int ret; 180358360b13Sdlg 180458360b13Sdlg wg_peer_get_endpoint(peer, &endpoint); 180558360b13Sdlg 180658360b13Sdlg while ((m = wg_queue_dequeue(&peer->p_encap_queue, &t)) != NULL) { 180758360b13Sdlg /* t_mbuf will contain the encrypted packet */ 180858360b13Sdlg if (t->t_mbuf == NULL){ 180958360b13Sdlg counters_inc(sc->sc_if.if_counters, ifc_oerrors); 181058360b13Sdlg m_freem(m); 181158360b13Sdlg continue; 181258360b13Sdlg } 181358360b13Sdlg 181458360b13Sdlg ret = wg_send(sc, &endpoint, t->t_mbuf); 181558360b13Sdlg 181658360b13Sdlg if (ret == 0) { 181758360b13Sdlg wg_timers_event_any_authenticated_packet_traversal( 181858360b13Sdlg &peer->p_timers); 181958360b13Sdlg wg_timers_event_any_authenticated_packet_sent( 182058360b13Sdlg &peer->p_timers); 182158360b13Sdlg 182258360b13Sdlg if (m->m_pkthdr.len != 0) 182358360b13Sdlg wg_timers_event_data_sent(&peer->p_timers); 182458360b13Sdlg } else if (ret == EADDRNOTAVAIL) { 182558360b13Sdlg wg_peer_clear_src(peer); 182658360b13Sdlg wg_peer_get_endpoint(peer, &endpoint); 182758360b13Sdlg } 182858360b13Sdlg 182958360b13Sdlg m_freem(m); 183058360b13Sdlg } 183158360b13Sdlg } 183258360b13Sdlg 183358360b13Sdlg void 183458360b13Sdlg wg_deliver_in(void *_peer) 183558360b13Sdlg { 183658360b13Sdlg struct wg_peer *peer = _peer; 183758360b13Sdlg struct wg_softc *sc = peer->p_sc; 183858360b13Sdlg struct wg_tag *t; 183958360b13Sdlg struct mbuf *m; 184058360b13Sdlg 184158360b13Sdlg while ((m = wg_queue_dequeue(&peer->p_decap_queue, &t)) != NULL) { 184258360b13Sdlg /* t_mbuf will contain the decrypted packet */ 184358360b13Sdlg if (t->t_mbuf == NULL) { 184458360b13Sdlg counters_inc(sc->sc_if.if_counters, ifc_ierrors); 184558360b13Sdlg m_freem(m); 184658360b13Sdlg continue; 184758360b13Sdlg } 184858360b13Sdlg 184958360b13Sdlg /* From here on m == t->t_mbuf */ 185058360b13Sdlg KASSERT(m == t->t_mbuf); 185158360b13Sdlg 185258360b13Sdlg wg_timers_event_any_authenticated_packet_received( 185358360b13Sdlg &peer->p_timers); 185458360b13Sdlg wg_timers_event_any_authenticated_packet_traversal( 185558360b13Sdlg &peer->p_timers); 185658360b13Sdlg 185758360b13Sdlg if (m->m_pkthdr.len == 0) { 185858360b13Sdlg m_freem(m); 185958360b13Sdlg continue; 186058360b13Sdlg } 186158360b13Sdlg 186258360b13Sdlg #if NBPFILTER > 0 186358360b13Sdlg if (sc->sc_if.if_bpf != NULL) 186458360b13Sdlg bpf_mtap_af(sc->sc_if.if_bpf, 186558360b13Sdlg m->m_pkthdr.ph_family, m, BPF_DIRECTION_IN); 186658360b13Sdlg #endif 186758360b13Sdlg 186858360b13Sdlg NET_LOCK(); 186958360b13Sdlg if (m->m_pkthdr.ph_family == AF_INET) 187058360b13Sdlg ipv4_input(&sc->sc_if, m); 187158360b13Sdlg #ifdef INET6 187258360b13Sdlg else if (m->m_pkthdr.ph_family == AF_INET6) 187358360b13Sdlg ipv6_input(&sc->sc_if, m); 187458360b13Sdlg #endif 187558360b13Sdlg else 187658360b13Sdlg panic("invalid ph_family"); 187758360b13Sdlg NET_UNLOCK(); 187858360b13Sdlg 187958360b13Sdlg wg_timers_event_data_received(&peer->p_timers); 188058360b13Sdlg } 188158360b13Sdlg } 188258360b13Sdlg 188358360b13Sdlg int 188458360b13Sdlg wg_queue_in(struct wg_softc *sc, struct wg_peer *peer, struct mbuf *m) 188558360b13Sdlg { 188658360b13Sdlg struct wg_ring *parallel = &sc->sc_decap_ring; 188758360b13Sdlg struct wg_queue *serial = &peer->p_decap_queue; 188858360b13Sdlg struct wg_tag *t; 188958360b13Sdlg 189058360b13Sdlg mtx_enter(&serial->q_mtx); 189158360b13Sdlg if (serial->q_list.ml_len < MAX_QUEUED_PKT) { 189258360b13Sdlg ml_enqueue(&serial->q_list, m); 189358360b13Sdlg mtx_leave(&serial->q_mtx); 189458360b13Sdlg } else { 189558360b13Sdlg mtx_leave(&serial->q_mtx); 189658360b13Sdlg m_freem(m); 189758360b13Sdlg return ENOBUFS; 189858360b13Sdlg } 189958360b13Sdlg 190058360b13Sdlg mtx_enter(¶llel->r_mtx); 190158360b13Sdlg if (parallel->r_tail - parallel->r_head < MAX_QUEUED_PKT) { 190258360b13Sdlg parallel->r_buf[parallel->r_tail & MAX_QUEUED_PKT_MASK] = m; 190358360b13Sdlg parallel->r_tail++; 190458360b13Sdlg mtx_leave(¶llel->r_mtx); 190558360b13Sdlg } else { 190658360b13Sdlg mtx_leave(¶llel->r_mtx); 190758360b13Sdlg t = wg_tag_get(m); 190858360b13Sdlg t->t_done = 1; 190958360b13Sdlg return ENOBUFS; 191058360b13Sdlg } 191158360b13Sdlg 191258360b13Sdlg return 0; 191358360b13Sdlg } 191458360b13Sdlg 191558360b13Sdlg void 191658360b13Sdlg wg_queue_out(struct wg_softc *sc, struct wg_peer *peer) 191758360b13Sdlg { 191858360b13Sdlg struct wg_ring *parallel = &sc->sc_encap_ring; 191958360b13Sdlg struct wg_queue *serial = &peer->p_encap_queue; 192058360b13Sdlg struct mbuf_list ml, ml_free; 192158360b13Sdlg struct mbuf *m; 192258360b13Sdlg struct wg_tag *t; 192358360b13Sdlg int dropped; 192458360b13Sdlg 192558360b13Sdlg /* 192658360b13Sdlg * We delist all staged packets and then add them to the queues. This 19278569c6f6Stobhe * can race with wg_qstart when called from wg_send_keepalive, however 19288569c6f6Stobhe * wg_qstart will not race as it is serialised. 192958360b13Sdlg */ 193058360b13Sdlg mq_delist(&peer->p_stage_queue, &ml); 193158360b13Sdlg ml_init(&ml_free); 193258360b13Sdlg 193358360b13Sdlg while ((m = ml_dequeue(&ml)) != NULL) { 193458360b13Sdlg mtx_enter(&serial->q_mtx); 193558360b13Sdlg if (serial->q_list.ml_len < MAX_QUEUED_PKT) { 193658360b13Sdlg ml_enqueue(&serial->q_list, m); 193758360b13Sdlg mtx_leave(&serial->q_mtx); 193858360b13Sdlg } else { 193958360b13Sdlg mtx_leave(&serial->q_mtx); 194058360b13Sdlg ml_enqueue(&ml_free, m); 194158360b13Sdlg continue; 194258360b13Sdlg } 194358360b13Sdlg 194458360b13Sdlg mtx_enter(¶llel->r_mtx); 194558360b13Sdlg if (parallel->r_tail - parallel->r_head < MAX_QUEUED_PKT) { 194658360b13Sdlg parallel->r_buf[parallel->r_tail & MAX_QUEUED_PKT_MASK] = m; 194758360b13Sdlg parallel->r_tail++; 194858360b13Sdlg mtx_leave(¶llel->r_mtx); 194958360b13Sdlg } else { 195058360b13Sdlg mtx_leave(¶llel->r_mtx); 195158360b13Sdlg t = wg_tag_get(m); 195258360b13Sdlg t->t_done = 1; 195358360b13Sdlg } 195458360b13Sdlg } 195558360b13Sdlg 195658360b13Sdlg if ((dropped = ml_purge(&ml_free)) > 0) 195758360b13Sdlg counters_add(sc->sc_if.if_counters, ifc_oqdrops, dropped); 195858360b13Sdlg } 195958360b13Sdlg 196058360b13Sdlg struct mbuf * 196158360b13Sdlg wg_ring_dequeue(struct wg_ring *r) 196258360b13Sdlg { 196358360b13Sdlg struct mbuf *m = NULL; 196458360b13Sdlg mtx_enter(&r->r_mtx); 196558360b13Sdlg if (r->r_head != r->r_tail) { 196658360b13Sdlg m = r->r_buf[r->r_head & MAX_QUEUED_PKT_MASK]; 196758360b13Sdlg r->r_head++; 196858360b13Sdlg } 196958360b13Sdlg mtx_leave(&r->r_mtx); 197058360b13Sdlg return m; 197158360b13Sdlg } 197258360b13Sdlg 197358360b13Sdlg struct mbuf * 197458360b13Sdlg wg_queue_dequeue(struct wg_queue *q, struct wg_tag **t) 197558360b13Sdlg { 197658360b13Sdlg struct mbuf *m; 197758360b13Sdlg mtx_enter(&q->q_mtx); 197858360b13Sdlg if ((m = q->q_list.ml_head) != NULL && (*t = wg_tag_get(m))->t_done) 197958360b13Sdlg ml_dequeue(&q->q_list); 198058360b13Sdlg else 198158360b13Sdlg m = NULL; 198258360b13Sdlg mtx_leave(&q->q_mtx); 198358360b13Sdlg return m; 198458360b13Sdlg } 198558360b13Sdlg 198658360b13Sdlg size_t 198758360b13Sdlg wg_queue_len(struct wg_queue *q) 198858360b13Sdlg { 198958360b13Sdlg size_t len; 199058360b13Sdlg mtx_enter(&q->q_mtx); 199158360b13Sdlg len = q->q_list.ml_len; 199258360b13Sdlg mtx_leave(&q->q_mtx); 199358360b13Sdlg return len; 199458360b13Sdlg } 199558360b13Sdlg 199658360b13Sdlg struct noise_remote * 199758360b13Sdlg wg_remote_get(void *_sc, uint8_t public[NOISE_PUBLIC_KEY_LEN]) 199858360b13Sdlg { 199958360b13Sdlg struct wg_peer *peer; 200058360b13Sdlg struct wg_softc *sc = _sc; 200158360b13Sdlg if ((peer = wg_peer_lookup(sc, public)) == NULL) 200258360b13Sdlg return NULL; 200358360b13Sdlg return &peer->p_remote; 200458360b13Sdlg } 200558360b13Sdlg 200658360b13Sdlg uint32_t 200758360b13Sdlg wg_index_set(void *_sc, struct noise_remote *remote) 200858360b13Sdlg { 200958360b13Sdlg struct wg_peer *peer; 201058360b13Sdlg struct wg_softc *sc = _sc; 201158360b13Sdlg struct wg_index *index, *iter; 201258360b13Sdlg uint32_t key; 201358360b13Sdlg 201458360b13Sdlg /* 201558360b13Sdlg * We can modify this without a lock as wg_index_set, wg_index_drop are 201658360b13Sdlg * guaranteed to be serialised (per remote). 201758360b13Sdlg */ 201858360b13Sdlg peer = CONTAINER_OF(remote, struct wg_peer, p_remote); 201958360b13Sdlg index = SLIST_FIRST(&peer->p_unused_index); 202058360b13Sdlg KASSERT(index != NULL); 202158360b13Sdlg SLIST_REMOVE_HEAD(&peer->p_unused_index, i_unused_entry); 202258360b13Sdlg 202358360b13Sdlg index->i_value = remote; 202458360b13Sdlg 202558360b13Sdlg mtx_enter(&sc->sc_index_mtx); 202658360b13Sdlg assign_id: 202758360b13Sdlg key = index->i_key = arc4random(); 202858360b13Sdlg key &= sc->sc_index_mask; 202958360b13Sdlg LIST_FOREACH(iter, &sc->sc_index[key], i_entry) 203058360b13Sdlg if (iter->i_key == index->i_key) 203158360b13Sdlg goto assign_id; 203258360b13Sdlg 203358360b13Sdlg LIST_INSERT_HEAD(&sc->sc_index[key], index, i_entry); 203458360b13Sdlg 203558360b13Sdlg mtx_leave(&sc->sc_index_mtx); 203658360b13Sdlg 203758360b13Sdlg /* Likewise, no need to lock for index here. */ 203858360b13Sdlg return index->i_key; 203958360b13Sdlg } 204058360b13Sdlg 204158360b13Sdlg struct noise_remote * 204258360b13Sdlg wg_index_get(void *_sc, uint32_t key0) 204358360b13Sdlg { 204458360b13Sdlg struct wg_softc *sc = _sc; 204558360b13Sdlg struct wg_index *iter; 204658360b13Sdlg struct noise_remote *remote = NULL; 204758360b13Sdlg uint32_t key = key0 & sc->sc_index_mask; 204858360b13Sdlg 204958360b13Sdlg mtx_enter(&sc->sc_index_mtx); 205058360b13Sdlg LIST_FOREACH(iter, &sc->sc_index[key], i_entry) 205158360b13Sdlg if (iter->i_key == key0) { 205258360b13Sdlg remote = iter->i_value; 205358360b13Sdlg break; 205458360b13Sdlg } 205558360b13Sdlg mtx_leave(&sc->sc_index_mtx); 205658360b13Sdlg return remote; 205758360b13Sdlg } 205858360b13Sdlg 205958360b13Sdlg void 206058360b13Sdlg wg_index_drop(void *_sc, uint32_t key0) 206158360b13Sdlg { 206258360b13Sdlg struct wg_softc *sc = _sc; 206358360b13Sdlg struct wg_index *iter; 206458360b13Sdlg struct wg_peer *peer = NULL; 206558360b13Sdlg uint32_t key = key0 & sc->sc_index_mask; 206658360b13Sdlg 206758360b13Sdlg mtx_enter(&sc->sc_index_mtx); 206858360b13Sdlg LIST_FOREACH(iter, &sc->sc_index[key], i_entry) 206958360b13Sdlg if (iter->i_key == key0) { 207058360b13Sdlg LIST_REMOVE(iter, i_entry); 207158360b13Sdlg break; 207258360b13Sdlg } 207358360b13Sdlg mtx_leave(&sc->sc_index_mtx); 207458360b13Sdlg 207558360b13Sdlg /* We expect a peer */ 207658360b13Sdlg peer = CONTAINER_OF(iter->i_value, struct wg_peer, p_remote); 207758360b13Sdlg KASSERT(peer != NULL); 207858360b13Sdlg SLIST_INSERT_HEAD(&peer->p_unused_index, iter, i_unused_entry); 207958360b13Sdlg } 208058360b13Sdlg 208158360b13Sdlg struct mbuf * 208258360b13Sdlg wg_input(void *_sc, struct mbuf *m, struct ip *ip, struct ip6_hdr *ip6, 208358360b13Sdlg void *_uh, int hlen) 208458360b13Sdlg { 208558360b13Sdlg struct wg_pkt_data *data; 208658360b13Sdlg struct noise_remote *remote; 208758360b13Sdlg struct wg_tag *t; 208858360b13Sdlg struct wg_softc *sc = _sc; 208958360b13Sdlg struct udphdr *uh = _uh; 2090*f08653c5Smvs char ipaddr[INET6_ADDRSTRLEN]; 209158360b13Sdlg 209258360b13Sdlg NET_ASSERT_LOCKED(); 209358360b13Sdlg 209458360b13Sdlg if ((t = wg_tag_get(m)) == NULL) { 209558360b13Sdlg m_freem(m); 209658360b13Sdlg return NULL; 209758360b13Sdlg } 209858360b13Sdlg 209958360b13Sdlg if (ip != NULL) { 210058360b13Sdlg t->t_endpoint.e_remote.r_sa.sa_len = sizeof(struct sockaddr_in); 210158360b13Sdlg t->t_endpoint.e_remote.r_sa.sa_family = AF_INET; 210258360b13Sdlg t->t_endpoint.e_remote.r_sin.sin_port = uh->uh_sport; 210358360b13Sdlg t->t_endpoint.e_remote.r_sin.sin_addr = ip->ip_src; 210458360b13Sdlg t->t_endpoint.e_local.l_in = ip->ip_dst; 210558360b13Sdlg #ifdef INET6 210658360b13Sdlg } else if (ip6 != NULL) { 210758360b13Sdlg t->t_endpoint.e_remote.r_sa.sa_len = sizeof(struct sockaddr_in6); 210858360b13Sdlg t->t_endpoint.e_remote.r_sa.sa_family = AF_INET6; 210958360b13Sdlg t->t_endpoint.e_remote.r_sin6.sin6_port = uh->uh_sport; 211058360b13Sdlg t->t_endpoint.e_remote.r_sin6.sin6_addr = ip6->ip6_src; 211158360b13Sdlg t->t_endpoint.e_local.l_in6 = ip6->ip6_dst; 211258360b13Sdlg #endif 211358360b13Sdlg } else { 211458360b13Sdlg m_freem(m); 211558360b13Sdlg return NULL; 211658360b13Sdlg } 211758360b13Sdlg 211858360b13Sdlg /* m has a IP/IPv6 header of hlen length, we don't need it anymore. */ 211958360b13Sdlg m_adj(m, hlen); 212058360b13Sdlg 2121dc736ae2Stb /* 2122dc736ae2Stb * Ensure mbuf is contiguous over full length of packet. This is done 2123c04adf2dSsthen * so we can directly read the handshake values in wg_handshake, and so 2124dc736ae2Stb * we can decrypt a transport packet by passing a single buffer to 2125dc736ae2Stb * noise_remote_decrypt in wg_decap. 2126dc736ae2Stb */ 2127dc736ae2Stb if ((m = m_pullup(m, m->m_pkthdr.len)) == NULL) 212858360b13Sdlg return NULL; 212958360b13Sdlg 213058360b13Sdlg if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) && 213158360b13Sdlg *mtod(m, uint32_t *) == WG_PKT_INITIATION) || 213258360b13Sdlg (m->m_pkthdr.len == sizeof(struct wg_pkt_response) && 213358360b13Sdlg *mtod(m, uint32_t *) == WG_PKT_RESPONSE) || 213458360b13Sdlg (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) && 213558360b13Sdlg *mtod(m, uint32_t *) == WG_PKT_COOKIE)) { 213658360b13Sdlg 213758360b13Sdlg if (mq_enqueue(&sc->sc_handshake_queue, m) != 0) 2138*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Dropping handshake" 2139*f08653c5Smvs "packet from %s\n", 2140*f08653c5Smvs sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr, 2141*f08653c5Smvs sizeof(ipaddr))); 214258360b13Sdlg task_add(wg_handshake_taskq, &sc->sc_handshake); 214358360b13Sdlg 214458360b13Sdlg } else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) + 214558360b13Sdlg NOISE_AUTHTAG_LEN && *mtod(m, uint32_t *) == WG_PKT_DATA) { 214658360b13Sdlg 214758360b13Sdlg data = mtod(m, struct wg_pkt_data *); 214858360b13Sdlg 214958360b13Sdlg if ((remote = wg_index_get(sc, data->r_idx)) != NULL) { 215058360b13Sdlg t->t_peer = CONTAINER_OF(remote, struct wg_peer, 215158360b13Sdlg p_remote); 215258360b13Sdlg t->t_mbuf = NULL; 215358360b13Sdlg t->t_done = 0; 215458360b13Sdlg 215558360b13Sdlg if (wg_queue_in(sc, t->t_peer, m) != 0) 215658360b13Sdlg counters_inc(sc->sc_if.if_counters, 215758360b13Sdlg ifc_iqdrops); 215858360b13Sdlg task_add(wg_crypt_taskq, &sc->sc_decap); 215958360b13Sdlg } else { 216058360b13Sdlg counters_inc(sc->sc_if.if_counters, ifc_ierrors); 216158360b13Sdlg m_freem(m); 216258360b13Sdlg } 216358360b13Sdlg } else { 216458360b13Sdlg counters_inc(sc->sc_if.if_counters, ifc_ierrors); 216558360b13Sdlg m_freem(m); 216658360b13Sdlg } 216758360b13Sdlg 216858360b13Sdlg return NULL; 216958360b13Sdlg } 217058360b13Sdlg 217158360b13Sdlg void 21728569c6f6Stobhe wg_qstart(struct ifqueue *ifq) 217358360b13Sdlg { 21748569c6f6Stobhe struct ifnet *ifp = ifq->ifq_if; 217558360b13Sdlg struct wg_softc *sc = ifp->if_softc; 217658360b13Sdlg struct wg_peer *peer; 217758360b13Sdlg struct wg_tag *t; 217858360b13Sdlg struct mbuf *m; 217958360b13Sdlg SLIST_HEAD(,wg_peer) start_list; 218058360b13Sdlg 218158360b13Sdlg SLIST_INIT(&start_list); 218258360b13Sdlg 218358360b13Sdlg /* 218458360b13Sdlg * We should be OK to modify p_start_list, p_start_onlist in this 21858569c6f6Stobhe * function as there should only be one ifp->if_qstart invoked at a 21868569c6f6Stobhe * time. 218758360b13Sdlg */ 21888569c6f6Stobhe while ((m = ifq_dequeue(ifq)) != NULL) { 218958360b13Sdlg t = wg_tag_get(m); 219058360b13Sdlg peer = t->t_peer; 219158360b13Sdlg if (mq_push(&peer->p_stage_queue, m) != 0) 219258360b13Sdlg counters_inc(ifp->if_counters, ifc_oqdrops); 219358360b13Sdlg if (!peer->p_start_onlist) { 219458360b13Sdlg SLIST_INSERT_HEAD(&start_list, peer, p_start_list); 219558360b13Sdlg peer->p_start_onlist = 1; 219658360b13Sdlg } 219758360b13Sdlg } 219858360b13Sdlg SLIST_FOREACH(peer, &start_list, p_start_list) { 219958360b13Sdlg if (noise_remote_ready(&peer->p_remote) == 0) 220058360b13Sdlg wg_queue_out(sc, peer); 220158360b13Sdlg else 220258360b13Sdlg wg_timers_event_want_initiation(&peer->p_timers); 220358360b13Sdlg peer->p_start_onlist = 0; 220458360b13Sdlg } 220558360b13Sdlg task_add(wg_crypt_taskq, &sc->sc_encap); 220658360b13Sdlg } 220758360b13Sdlg 220858360b13Sdlg int 220958360b13Sdlg wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa, 221058360b13Sdlg struct rtentry *rt) 221158360b13Sdlg { 221258360b13Sdlg struct wg_softc *sc = ifp->if_softc; 221358360b13Sdlg struct wg_peer *peer; 221458360b13Sdlg struct wg_tag *t; 221558360b13Sdlg int af, ret = EINVAL; 221658360b13Sdlg 221758360b13Sdlg NET_ASSERT_LOCKED(); 221858360b13Sdlg 221958360b13Sdlg if ((t = wg_tag_get(m)) == NULL) { 222058360b13Sdlg ret = ENOBUFS; 222158360b13Sdlg goto error; 222258360b13Sdlg } 222358360b13Sdlg 222458360b13Sdlg m->m_pkthdr.ph_family = sa->sa_family; 222558360b13Sdlg if (sa->sa_family == AF_INET) { 222658360b13Sdlg peer = wg_aip_lookup(sc->sc_aip4, 222758360b13Sdlg &mtod(m, struct ip *)->ip_dst); 222858360b13Sdlg #ifdef INET6 222958360b13Sdlg } else if (sa->sa_family == AF_INET6) { 223058360b13Sdlg peer = wg_aip_lookup(sc->sc_aip6, 223158360b13Sdlg &mtod(m, struct ip6_hdr *)->ip6_dst); 223258360b13Sdlg #endif 223358360b13Sdlg } else { 223458360b13Sdlg ret = EAFNOSUPPORT; 223558360b13Sdlg goto error; 223658360b13Sdlg } 223758360b13Sdlg 223858360b13Sdlg #if NBPFILTER > 0 223958360b13Sdlg if (sc->sc_if.if_bpf) 224058360b13Sdlg bpf_mtap_af(sc->sc_if.if_bpf, sa->sa_family, m, 224158360b13Sdlg BPF_DIRECTION_OUT); 224258360b13Sdlg #endif 224358360b13Sdlg 224458360b13Sdlg if (peer == NULL) { 224558360b13Sdlg ret = ENETUNREACH; 224658360b13Sdlg goto error; 224758360b13Sdlg } 224858360b13Sdlg 224958360b13Sdlg af = peer->p_endpoint.e_remote.r_sa.sa_family; 225058360b13Sdlg if (af != AF_INET && af != AF_INET6) { 2251*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "No valid endpoint has been " 2252*f08653c5Smvs "configured or discovered for peer %llu\n", peer->p_id); 225358360b13Sdlg ret = EDESTADDRREQ; 225458360b13Sdlg goto error; 225558360b13Sdlg } 225658360b13Sdlg 225758360b13Sdlg if (m->m_pkthdr.ph_loopcnt++ > M_MAXLOOP) { 2258*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "Packet looped\n"); 225958360b13Sdlg ret = ELOOP; 226058360b13Sdlg goto error; 226158360b13Sdlg } 226258360b13Sdlg 226358360b13Sdlg /* 226458360b13Sdlg * As we hold a reference to peer in the mbuf, we can't handle a 226558360b13Sdlg * delayed packet without doing some refcnting. If a peer is removed 226658360b13Sdlg * while a delayed holds a reference, bad things will happen. For the 226758360b13Sdlg * time being, delayed packets are unsupported. This may be fixed with 22688569c6f6Stobhe * another aip_lookup in wg_qstart, or refcnting as mentioned before. 226958360b13Sdlg */ 227058360b13Sdlg if (m->m_pkthdr.pf.delay > 0) { 2271*f08653c5Smvs WGPRINTF(LOG_DEBUG, sc, NULL, "PF delay unsupported\n"); 227258360b13Sdlg ret = EOPNOTSUPP; 227358360b13Sdlg goto error; 227458360b13Sdlg } 227558360b13Sdlg 227658360b13Sdlg t->t_peer = peer; 227758360b13Sdlg t->t_mbuf = NULL; 227858360b13Sdlg t->t_done = 0; 227958360b13Sdlg t->t_mtu = ifp->if_mtu; 228058360b13Sdlg 228158360b13Sdlg /* 228258360b13Sdlg * We still have an issue with ifq that will count a packet that gets 22838569c6f6Stobhe * dropped in wg_qstart, or not encrypted. These get counted as 228458360b13Sdlg * ofails or oqdrops, so the packet gets counted twice. 228558360b13Sdlg */ 228658360b13Sdlg return if_enqueue(ifp, m); 228758360b13Sdlg error: 228858360b13Sdlg counters_inc(ifp->if_counters, ifc_oerrors); 228958360b13Sdlg m_freem(m); 229058360b13Sdlg return ret; 229158360b13Sdlg } 229258360b13Sdlg 229358360b13Sdlg int 229458360b13Sdlg wg_ioctl_set(struct wg_softc *sc, struct wg_data_io *data) 229558360b13Sdlg { 229658360b13Sdlg struct wg_interface_io *iface_p, iface_o; 229758360b13Sdlg struct wg_peer_io *peer_p, peer_o; 229858360b13Sdlg struct wg_aip_io *aip_p, aip_o; 229958360b13Sdlg 230058360b13Sdlg struct wg_peer *peer, *tpeer; 230158360b13Sdlg struct wg_aip *aip, *taip; 230258360b13Sdlg 230358360b13Sdlg in_port_t port; 230458360b13Sdlg int rtable; 230558360b13Sdlg 230658360b13Sdlg uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE]; 230758360b13Sdlg size_t i, j; 230858360b13Sdlg int ret, has_identity; 230958360b13Sdlg 231058360b13Sdlg if ((ret = suser(curproc)) != 0) 231158360b13Sdlg return ret; 231258360b13Sdlg 231358360b13Sdlg rw_enter_write(&sc->sc_lock); 231458360b13Sdlg 231558360b13Sdlg iface_p = data->wgd_interface; 231658360b13Sdlg if ((ret = copyin(iface_p, &iface_o, sizeof(iface_o))) != 0) 231758360b13Sdlg goto error; 231858360b13Sdlg 231958360b13Sdlg if (iface_o.i_flags & WG_INTERFACE_REPLACE_PEERS) 232058360b13Sdlg TAILQ_FOREACH_SAFE(peer, &sc->sc_peer_seq, p_seq_entry, tpeer) 232158360b13Sdlg wg_peer_destroy(peer); 232258360b13Sdlg 232358360b13Sdlg if (iface_o.i_flags & WG_INTERFACE_HAS_PRIVATE && 232458360b13Sdlg (noise_local_keys(&sc->sc_local, NULL, private) || 232558360b13Sdlg timingsafe_bcmp(private, iface_o.i_private, WG_KEY_SIZE))) { 232658360b13Sdlg if (curve25519_generate_public(public, iface_o.i_private)) { 232758360b13Sdlg if ((peer = wg_peer_lookup(sc, public)) != NULL) 232858360b13Sdlg wg_peer_destroy(peer); 232958360b13Sdlg } 233058360b13Sdlg noise_local_lock_identity(&sc->sc_local); 233158360b13Sdlg has_identity = noise_local_set_private(&sc->sc_local, 233258360b13Sdlg iface_o.i_private); 233358360b13Sdlg TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) { 233458360b13Sdlg noise_remote_precompute(&peer->p_remote); 233558360b13Sdlg wg_timers_event_reset_handshake_last_sent(&peer->p_timers); 233658360b13Sdlg noise_remote_expire_current(&peer->p_remote); 233758360b13Sdlg } 233858360b13Sdlg cookie_checker_update(&sc->sc_cookie, 233958360b13Sdlg has_identity == 0 ? public : NULL); 234058360b13Sdlg noise_local_unlock_identity(&sc->sc_local); 234158360b13Sdlg } 234258360b13Sdlg 234358360b13Sdlg if (iface_o.i_flags & WG_INTERFACE_HAS_PORT) 234458360b13Sdlg port = htons(iface_o.i_port); 234558360b13Sdlg else 234658360b13Sdlg port = sc->sc_udp_port; 234758360b13Sdlg 234858360b13Sdlg if (iface_o.i_flags & WG_INTERFACE_HAS_RTABLE) 234958360b13Sdlg rtable = iface_o.i_rtable; 235058360b13Sdlg else 235158360b13Sdlg rtable = sc->sc_udp_rtable; 235258360b13Sdlg 235358360b13Sdlg if (port != sc->sc_udp_port || rtable != sc->sc_udp_rtable) { 235458360b13Sdlg TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) 235558360b13Sdlg wg_peer_clear_src(peer); 235658360b13Sdlg 235758360b13Sdlg if (sc->sc_if.if_flags & IFF_RUNNING) 235858360b13Sdlg if ((ret = wg_bind(sc, &port, &rtable)) != 0) 235958360b13Sdlg goto error; 236058360b13Sdlg 236158360b13Sdlg sc->sc_udp_port = port; 236258360b13Sdlg sc->sc_udp_rtable = rtable; 236358360b13Sdlg } 236458360b13Sdlg 236558360b13Sdlg peer_p = &iface_p->i_peers[0]; 236658360b13Sdlg for (i = 0; i < iface_o.i_peers_count; i++) { 236758360b13Sdlg if ((ret = copyin(peer_p, &peer_o, sizeof(peer_o))) != 0) 236858360b13Sdlg goto error; 236958360b13Sdlg 237058360b13Sdlg /* Peer must have public key */ 237158360b13Sdlg if (!(peer_o.p_flags & WG_PEER_HAS_PUBLIC)) 2372a86e8a3fSyasuoka goto next_peer; 237358360b13Sdlg 237458360b13Sdlg /* 0 = latest protocol, 1 = this protocol */ 237558360b13Sdlg if (peer_o.p_protocol_version != 0) { 237658360b13Sdlg if (peer_o.p_protocol_version > 1) { 237758360b13Sdlg ret = EPFNOSUPPORT; 237858360b13Sdlg goto error; 237958360b13Sdlg } 238058360b13Sdlg } 238158360b13Sdlg 238258360b13Sdlg /* Get local public and check that peer key doesn't match */ 238358360b13Sdlg if (noise_local_keys(&sc->sc_local, public, NULL) == 0 && 238458360b13Sdlg bcmp(public, peer_o.p_public, WG_KEY_SIZE) == 0) 2385a86e8a3fSyasuoka goto next_peer; 238658360b13Sdlg 238758360b13Sdlg /* Lookup peer, or create if it doesn't exist */ 238858360b13Sdlg if ((peer = wg_peer_lookup(sc, peer_o.p_public)) == NULL) { 238958360b13Sdlg /* If we want to delete, no need creating a new one. 239058360b13Sdlg * Also, don't create a new one if we only want to 239158360b13Sdlg * update. */ 239258360b13Sdlg if (peer_o.p_flags & (WG_PEER_REMOVE|WG_PEER_UPDATE)) 2393a86e8a3fSyasuoka goto next_peer; 239458360b13Sdlg 239558360b13Sdlg if ((peer = wg_peer_create(sc, 239658360b13Sdlg peer_o.p_public)) == NULL) { 239758360b13Sdlg ret = ENOMEM; 239858360b13Sdlg goto error; 239958360b13Sdlg } 240058360b13Sdlg } 240158360b13Sdlg 240258360b13Sdlg /* Remove peer and continue if specified */ 240358360b13Sdlg if (peer_o.p_flags & WG_PEER_REMOVE) { 240458360b13Sdlg wg_peer_destroy(peer); 2405a86e8a3fSyasuoka goto next_peer; 240658360b13Sdlg } 240758360b13Sdlg 240858360b13Sdlg if (peer_o.p_flags & WG_PEER_HAS_ENDPOINT) 240958360b13Sdlg wg_peer_set_sockaddr(peer, &peer_o.p_sa); 241058360b13Sdlg 241158360b13Sdlg if (peer_o.p_flags & WG_PEER_HAS_PSK) 241258360b13Sdlg noise_remote_set_psk(&peer->p_remote, peer_o.p_psk); 241358360b13Sdlg 241458360b13Sdlg if (peer_o.p_flags & WG_PEER_HAS_PKA) 241558360b13Sdlg wg_timers_set_persistent_keepalive(&peer->p_timers, 241658360b13Sdlg peer_o.p_pka); 241758360b13Sdlg 241858360b13Sdlg if (peer_o.p_flags & WG_PEER_REPLACE_AIPS) { 241958360b13Sdlg LIST_FOREACH_SAFE(aip, &peer->p_aip, a_entry, taip) { 242058360b13Sdlg wg_aip_remove(sc, peer, &aip->a_data); 242158360b13Sdlg } 242258360b13Sdlg } 242358360b13Sdlg 242411d9cfa5Skn if (peer_o.p_flags & WG_PEER_SET_DESCRIPTION) 242511d9cfa5Skn strlcpy(peer->p_description, peer_o.p_description, 242611d9cfa5Skn IFDESCRSIZE); 242711d9cfa5Skn 242858360b13Sdlg aip_p = &peer_p->p_aips[0]; 242958360b13Sdlg for (j = 0; j < peer_o.p_aips_count; j++) { 243058360b13Sdlg if ((ret = copyin(aip_p, &aip_o, sizeof(aip_o))) != 0) 243158360b13Sdlg goto error; 243258360b13Sdlg ret = wg_aip_add(sc, peer, &aip_o); 243358360b13Sdlg if (ret != 0) 243458360b13Sdlg goto error; 243558360b13Sdlg aip_p++; 243658360b13Sdlg } 243758360b13Sdlg 243858360b13Sdlg peer_p = (struct wg_peer_io *)aip_p; 2439a86e8a3fSyasuoka continue; 2440a86e8a3fSyasuoka next_peer: 2441a86e8a3fSyasuoka aip_p = &peer_p->p_aips[0]; 2442a86e8a3fSyasuoka aip_p += peer_o.p_aips_count; 2443a86e8a3fSyasuoka peer_p = (struct wg_peer_io *)aip_p; 244458360b13Sdlg } 244558360b13Sdlg 244658360b13Sdlg error: 244758360b13Sdlg rw_exit_write(&sc->sc_lock); 244858360b13Sdlg explicit_bzero(&iface_o, sizeof(iface_o)); 244958360b13Sdlg explicit_bzero(&peer_o, sizeof(peer_o)); 245058360b13Sdlg explicit_bzero(&aip_o, sizeof(aip_o)); 245158360b13Sdlg explicit_bzero(public, sizeof(public)); 245258360b13Sdlg explicit_bzero(private, sizeof(private)); 245358360b13Sdlg return ret; 245458360b13Sdlg } 245558360b13Sdlg 245658360b13Sdlg int 245758360b13Sdlg wg_ioctl_get(struct wg_softc *sc, struct wg_data_io *data) 245858360b13Sdlg { 245958360b13Sdlg struct wg_interface_io *iface_p, iface_o; 246058360b13Sdlg struct wg_peer_io *peer_p, peer_o; 246158360b13Sdlg struct wg_aip_io *aip_p; 246258360b13Sdlg 246358360b13Sdlg struct wg_peer *peer; 246458360b13Sdlg struct wg_aip *aip; 246558360b13Sdlg 246658360b13Sdlg size_t size, peer_count, aip_count; 246758360b13Sdlg int ret = 0, is_suser = suser(curproc) == 0; 246858360b13Sdlg 246958360b13Sdlg size = sizeof(struct wg_interface_io); 247058360b13Sdlg if (data->wgd_size < size && !is_suser) 247158360b13Sdlg goto ret_size; 247258360b13Sdlg 247358360b13Sdlg iface_p = data->wgd_interface; 247458360b13Sdlg bzero(&iface_o, sizeof(iface_o)); 247558360b13Sdlg 247658360b13Sdlg rw_enter_read(&sc->sc_lock); 247758360b13Sdlg 247858360b13Sdlg if (sc->sc_udp_port != 0) { 247958360b13Sdlg iface_o.i_port = ntohs(sc->sc_udp_port); 248058360b13Sdlg iface_o.i_flags |= WG_INTERFACE_HAS_PORT; 248158360b13Sdlg } 248258360b13Sdlg 248358360b13Sdlg if (sc->sc_udp_rtable != 0) { 248458360b13Sdlg iface_o.i_rtable = sc->sc_udp_rtable; 248558360b13Sdlg iface_o.i_flags |= WG_INTERFACE_HAS_RTABLE; 248658360b13Sdlg } 248758360b13Sdlg 248858360b13Sdlg if (!is_suser) 248958360b13Sdlg goto copy_out_iface; 249058360b13Sdlg 249158360b13Sdlg if (noise_local_keys(&sc->sc_local, iface_o.i_public, 249258360b13Sdlg iface_o.i_private) == 0) { 249358360b13Sdlg iface_o.i_flags |= WG_INTERFACE_HAS_PUBLIC; 249458360b13Sdlg iface_o.i_flags |= WG_INTERFACE_HAS_PRIVATE; 249558360b13Sdlg } 249658360b13Sdlg 249758360b13Sdlg size += sizeof(struct wg_peer_io) * sc->sc_peer_num; 249858360b13Sdlg size += sizeof(struct wg_aip_io) * sc->sc_aip_num; 249958360b13Sdlg if (data->wgd_size < size) 250058360b13Sdlg goto unlock_and_ret_size; 250158360b13Sdlg 250258360b13Sdlg peer_count = 0; 250358360b13Sdlg peer_p = &iface_p->i_peers[0]; 250458360b13Sdlg TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) { 250558360b13Sdlg bzero(&peer_o, sizeof(peer_o)); 250658360b13Sdlg peer_o.p_flags = WG_PEER_HAS_PUBLIC; 250758360b13Sdlg peer_o.p_protocol_version = 1; 250858360b13Sdlg 250958360b13Sdlg if (noise_remote_keys(&peer->p_remote, peer_o.p_public, 251058360b13Sdlg peer_o.p_psk) == 0) 251158360b13Sdlg peer_o.p_flags |= WG_PEER_HAS_PSK; 251258360b13Sdlg 251358360b13Sdlg if (wg_timers_get_persistent_keepalive(&peer->p_timers, 251458360b13Sdlg &peer_o.p_pka) == 0) 251558360b13Sdlg peer_o.p_flags |= WG_PEER_HAS_PKA; 251658360b13Sdlg 251758360b13Sdlg if (wg_peer_get_sockaddr(peer, &peer_o.p_sa) == 0) 251858360b13Sdlg peer_o.p_flags |= WG_PEER_HAS_ENDPOINT; 251958360b13Sdlg 252058360b13Sdlg mtx_enter(&peer->p_counters_mtx); 252158360b13Sdlg peer_o.p_txbytes = peer->p_counters_tx; 252258360b13Sdlg peer_o.p_rxbytes = peer->p_counters_rx; 252358360b13Sdlg mtx_leave(&peer->p_counters_mtx); 252458360b13Sdlg 252558360b13Sdlg wg_timers_get_last_handshake(&peer->p_timers, 252658360b13Sdlg &peer_o.p_last_handshake); 252758360b13Sdlg 252858360b13Sdlg aip_count = 0; 252958360b13Sdlg aip_p = &peer_p->p_aips[0]; 253058360b13Sdlg LIST_FOREACH(aip, &peer->p_aip, a_entry) { 253158360b13Sdlg if ((ret = copyout(&aip->a_data, aip_p, sizeof(*aip_p))) != 0) 253258360b13Sdlg goto unlock_and_ret_size; 253358360b13Sdlg aip_p++; 253458360b13Sdlg aip_count++; 253558360b13Sdlg } 253658360b13Sdlg peer_o.p_aips_count = aip_count; 253758360b13Sdlg 253811d9cfa5Skn strlcpy(peer_o.p_description, peer->p_description, IFDESCRSIZE); 253911d9cfa5Skn 254058360b13Sdlg if ((ret = copyout(&peer_o, peer_p, sizeof(peer_o))) != 0) 254158360b13Sdlg goto unlock_and_ret_size; 254258360b13Sdlg 254358360b13Sdlg peer_p = (struct wg_peer_io *)aip_p; 254458360b13Sdlg peer_count++; 254558360b13Sdlg } 254658360b13Sdlg iface_o.i_peers_count = peer_count; 254758360b13Sdlg 254858360b13Sdlg copy_out_iface: 254958360b13Sdlg ret = copyout(&iface_o, iface_p, sizeof(iface_o)); 255058360b13Sdlg unlock_and_ret_size: 255158360b13Sdlg rw_exit_read(&sc->sc_lock); 255258360b13Sdlg explicit_bzero(&iface_o, sizeof(iface_o)); 255358360b13Sdlg explicit_bzero(&peer_o, sizeof(peer_o)); 255458360b13Sdlg ret_size: 255558360b13Sdlg data->wgd_size = size; 255658360b13Sdlg return ret; 255758360b13Sdlg } 255858360b13Sdlg 255958360b13Sdlg int 256058360b13Sdlg wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 256158360b13Sdlg { 256258360b13Sdlg struct ifreq *ifr = (struct ifreq *) data; 256358360b13Sdlg struct wg_softc *sc = ifp->if_softc; 256458360b13Sdlg int ret = 0; 256558360b13Sdlg 256658360b13Sdlg switch (cmd) { 256758360b13Sdlg case SIOCSWG: 25682646c3b9Sclaudio NET_UNLOCK(); 256958360b13Sdlg ret = wg_ioctl_set(sc, (struct wg_data_io *) data); 25702646c3b9Sclaudio NET_LOCK(); 257158360b13Sdlg break; 257258360b13Sdlg case SIOCGWG: 25732646c3b9Sclaudio NET_UNLOCK(); 257458360b13Sdlg ret = wg_ioctl_get(sc, (struct wg_data_io *) data); 25752646c3b9Sclaudio NET_LOCK(); 257658360b13Sdlg break; 257758360b13Sdlg /* Interface IOCTLs */ 257858360b13Sdlg case SIOCSIFADDR: 257958360b13Sdlg SET(ifp->if_flags, IFF_UP); 258058360b13Sdlg /* FALLTHROUGH */ 258158360b13Sdlg case SIOCSIFFLAGS: 258258360b13Sdlg if (ISSET(ifp->if_flags, IFF_UP)) 258358360b13Sdlg ret = wg_up(sc); 258458360b13Sdlg else 258558360b13Sdlg wg_down(sc); 258658360b13Sdlg break; 258758360b13Sdlg case SIOCSIFMTU: 258858360b13Sdlg /* Arbitrary limits */ 258958360b13Sdlg if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > 9000) 259058360b13Sdlg ret = EINVAL; 259158360b13Sdlg else 259258360b13Sdlg ifp->if_mtu = ifr->ifr_mtu; 259358360b13Sdlg break; 259458360b13Sdlg case SIOCADDMULTI: 259558360b13Sdlg case SIOCDELMULTI: 259658360b13Sdlg break; 259758360b13Sdlg default: 259858360b13Sdlg ret = ENOTTY; 259958360b13Sdlg } 260058360b13Sdlg 260158360b13Sdlg return ret; 260258360b13Sdlg } 260358360b13Sdlg 260458360b13Sdlg int 260558360b13Sdlg wg_up(struct wg_softc *sc) 260658360b13Sdlg { 260758360b13Sdlg struct wg_peer *peer; 260858360b13Sdlg int ret = 0; 260958360b13Sdlg 261058360b13Sdlg NET_ASSERT_LOCKED(); 261158360b13Sdlg /* 261258360b13Sdlg * We use IFF_RUNNING as an exclusive access here. We also may want 261358360b13Sdlg * an exclusive sc_lock as wg_bind may write to sc_udp_port. We also 261458360b13Sdlg * want to drop NET_LOCK as we want to call socreate, sobind, etc. Once 261558360b13Sdlg * solock is no longer === NET_LOCK, we may be able to avoid this. 261658360b13Sdlg */ 261758360b13Sdlg if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)) { 261858360b13Sdlg SET(sc->sc_if.if_flags, IFF_RUNNING); 261958360b13Sdlg NET_UNLOCK(); 262058360b13Sdlg 262158360b13Sdlg rw_enter_write(&sc->sc_lock); 262258360b13Sdlg /* 262358360b13Sdlg * If we successfully bind the socket, then enable the timers 262458360b13Sdlg * for the peer. This will send all staged packets and a 262558360b13Sdlg * keepalive if necessary. 262658360b13Sdlg */ 262758360b13Sdlg ret = wg_bind(sc, &sc->sc_udp_port, &sc->sc_udp_rtable); 262858360b13Sdlg if (ret == 0) { 262958360b13Sdlg TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) { 263058360b13Sdlg wg_timers_enable(&peer->p_timers); 263158360b13Sdlg wg_queue_out(sc, peer); 263258360b13Sdlg } 263358360b13Sdlg } 263458360b13Sdlg rw_exit_write(&sc->sc_lock); 263558360b13Sdlg 263658360b13Sdlg NET_LOCK(); 263758360b13Sdlg if (ret != 0) 263858360b13Sdlg CLR(sc->sc_if.if_flags, IFF_RUNNING); 263958360b13Sdlg } 264058360b13Sdlg return ret; 264158360b13Sdlg } 264258360b13Sdlg 264358360b13Sdlg void 264458360b13Sdlg wg_down(struct wg_softc *sc) 264558360b13Sdlg { 264658360b13Sdlg struct wg_peer *peer; 264758360b13Sdlg 264858360b13Sdlg NET_ASSERT_LOCKED(); 264958360b13Sdlg if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)) 265058360b13Sdlg return; 265158360b13Sdlg CLR(sc->sc_if.if_flags, IFF_RUNNING); 265258360b13Sdlg NET_UNLOCK(); 265358360b13Sdlg 265458360b13Sdlg /* 265558360b13Sdlg * We only need a read lock here, as we aren't writing to anything 265658360b13Sdlg * that isn't granularly locked. 265758360b13Sdlg */ 265858360b13Sdlg rw_enter_read(&sc->sc_lock); 265958360b13Sdlg TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) { 266058360b13Sdlg mq_purge(&peer->p_stage_queue); 266158360b13Sdlg wg_timers_disable(&peer->p_timers); 266258360b13Sdlg } 266358360b13Sdlg 266458360b13Sdlg taskq_barrier(wg_handshake_taskq); 266558360b13Sdlg TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) { 266658360b13Sdlg noise_remote_clear(&peer->p_remote); 266758360b13Sdlg wg_timers_event_reset_handshake_last_sent(&peer->p_timers); 266858360b13Sdlg } 266958360b13Sdlg 267058360b13Sdlg wg_unbind(sc); 267158360b13Sdlg rw_exit_read(&sc->sc_lock); 267258360b13Sdlg NET_LOCK(); 267358360b13Sdlg } 267458360b13Sdlg 267558360b13Sdlg int 267658360b13Sdlg wg_clone_create(struct if_clone *ifc, int unit) 267758360b13Sdlg { 267858360b13Sdlg struct ifnet *ifp; 267958360b13Sdlg struct wg_softc *sc; 268058360b13Sdlg struct noise_upcall local_upcall; 268158360b13Sdlg 268258360b13Sdlg KERNEL_ASSERT_LOCKED(); 268358360b13Sdlg 268458360b13Sdlg if (wg_counter == 0) { 268558360b13Sdlg wg_handshake_taskq = taskq_create("wg_handshake", 268658360b13Sdlg 2, IPL_NET, TASKQ_MPSAFE); 268758360b13Sdlg wg_crypt_taskq = taskq_create("wg_crypt", 268858360b13Sdlg ncpus, IPL_NET, TASKQ_MPSAFE); 268958360b13Sdlg 269058360b13Sdlg if (wg_handshake_taskq == NULL || wg_crypt_taskq == NULL) { 269158360b13Sdlg if (wg_handshake_taskq != NULL) 269258360b13Sdlg taskq_destroy(wg_handshake_taskq); 269358360b13Sdlg if (wg_crypt_taskq != NULL) 269458360b13Sdlg taskq_destroy(wg_crypt_taskq); 269558360b13Sdlg wg_handshake_taskq = NULL; 269658360b13Sdlg wg_crypt_taskq = NULL; 269758360b13Sdlg return ENOTRECOVERABLE; 269858360b13Sdlg } 269958360b13Sdlg } 270058360b13Sdlg wg_counter++; 270158360b13Sdlg 270258360b13Sdlg if ((sc = malloc(sizeof(*sc), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) 270358360b13Sdlg goto ret_00; 270458360b13Sdlg 270558360b13Sdlg local_upcall.u_arg = sc; 270658360b13Sdlg local_upcall.u_remote_get = wg_remote_get; 270758360b13Sdlg local_upcall.u_index_set = wg_index_set; 270858360b13Sdlg local_upcall.u_index_drop = wg_index_drop; 270958360b13Sdlg 271058360b13Sdlg TAILQ_INIT(&sc->sc_peer_seq); 271158360b13Sdlg 271258360b13Sdlg /* sc_if is initialised after everything else */ 271358360b13Sdlg arc4random_buf(&sc->sc_secret, sizeof(sc->sc_secret)); 271458360b13Sdlg 271558360b13Sdlg rw_init(&sc->sc_lock, "wg"); 271658360b13Sdlg noise_local_init(&sc->sc_local, &local_upcall); 271758360b13Sdlg if (cookie_checker_init(&sc->sc_cookie, &wg_ratelimit_pool) != 0) 271858360b13Sdlg goto ret_01; 271958360b13Sdlg sc->sc_udp_port = 0; 272058360b13Sdlg sc->sc_udp_rtable = 0; 272158360b13Sdlg 272258360b13Sdlg rw_init(&sc->sc_so_lock, "wg_so"); 272358360b13Sdlg sc->sc_so4 = NULL; 272458360b13Sdlg #ifdef INET6 272558360b13Sdlg sc->sc_so6 = NULL; 272658360b13Sdlg #endif 272758360b13Sdlg 272858360b13Sdlg sc->sc_aip_num = 0; 272958360b13Sdlg if ((sc->sc_aip4 = art_alloc(0, 32, 0)) == NULL) 273058360b13Sdlg goto ret_02; 273158360b13Sdlg #ifdef INET6 273258360b13Sdlg if ((sc->sc_aip6 = art_alloc(0, 128, 0)) == NULL) 273358360b13Sdlg goto ret_03; 273458360b13Sdlg #endif 273558360b13Sdlg 273658360b13Sdlg rw_init(&sc->sc_peer_lock, "wg_peer"); 273758360b13Sdlg sc->sc_peer_num = 0; 273858360b13Sdlg if ((sc->sc_peer = hashinit(HASHTABLE_PEER_SIZE, M_DEVBUF, 273958360b13Sdlg M_NOWAIT, &sc->sc_peer_mask)) == NULL) 274058360b13Sdlg goto ret_04; 274158360b13Sdlg 274258360b13Sdlg mtx_init(&sc->sc_index_mtx, IPL_NET); 274358360b13Sdlg if ((sc->sc_index = hashinit(HASHTABLE_INDEX_SIZE, M_DEVBUF, 274458360b13Sdlg M_NOWAIT, &sc->sc_index_mask)) == NULL) 274558360b13Sdlg goto ret_05; 274658360b13Sdlg 274758360b13Sdlg task_set(&sc->sc_handshake, wg_handshake_worker, sc); 274858360b13Sdlg mq_init(&sc->sc_handshake_queue, MAX_QUEUED_HANDSHAKES, IPL_NET); 274958360b13Sdlg 275058360b13Sdlg task_set(&sc->sc_encap, wg_encap_worker, sc); 275158360b13Sdlg task_set(&sc->sc_decap, wg_decap_worker, sc); 275258360b13Sdlg 275358360b13Sdlg bzero(&sc->sc_encap_ring, sizeof(sc->sc_encap_ring)); 275458360b13Sdlg mtx_init(&sc->sc_encap_ring.r_mtx, IPL_NET); 275558360b13Sdlg bzero(&sc->sc_decap_ring, sizeof(sc->sc_decap_ring)); 275658360b13Sdlg mtx_init(&sc->sc_decap_ring.r_mtx, IPL_NET); 275758360b13Sdlg 275858360b13Sdlg /* We've setup the softc, now we can setup the ifnet */ 275958360b13Sdlg ifp = &sc->sc_if; 276058360b13Sdlg ifp->if_softc = sc; 276158360b13Sdlg 276258360b13Sdlg snprintf(ifp->if_xname, sizeof(ifp->if_xname), "wg%d", unit); 276358360b13Sdlg 276458360b13Sdlg ifp->if_mtu = DEFAULT_MTU; 276558360b13Sdlg ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_NOARP; 27668569c6f6Stobhe ifp->if_xflags = IFXF_CLONED | IFXF_MPSAFE; 27673e93ec8dStobhe ifp->if_txmit = 64; /* Keep our workers active for longer. */ 276858360b13Sdlg 276958360b13Sdlg ifp->if_ioctl = wg_ioctl; 27708569c6f6Stobhe ifp->if_qstart = wg_qstart; 277158360b13Sdlg ifp->if_output = wg_output; 277258360b13Sdlg 277358360b13Sdlg ifp->if_type = IFT_WIREGUARD; 277433947fcbSkn ifp->if_rtrequest = p2p_rtrequest; 277558360b13Sdlg 2776b5b8bc53Smvs if_counters_alloc(ifp); 277758360b13Sdlg if_attach(ifp); 277858360b13Sdlg if_alloc_sadl(ifp); 277958360b13Sdlg 278058360b13Sdlg #if NBPFILTER > 0 278158360b13Sdlg bpfattach(&ifp->if_bpf, ifp, DLT_LOOP, sizeof(uint32_t)); 278258360b13Sdlg #endif 278358360b13Sdlg 2784*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Interface created\n"); 278558360b13Sdlg 278658360b13Sdlg return 0; 278758360b13Sdlg ret_05: 278858360b13Sdlg hashfree(sc->sc_peer, HASHTABLE_PEER_SIZE, M_DEVBUF); 278958360b13Sdlg ret_04: 279058360b13Sdlg #ifdef INET6 279158360b13Sdlg free(sc->sc_aip6, M_RTABLE, sizeof(*sc->sc_aip6)); 279258360b13Sdlg ret_03: 279358360b13Sdlg #endif 279458360b13Sdlg free(sc->sc_aip4, M_RTABLE, sizeof(*sc->sc_aip4)); 279558360b13Sdlg ret_02: 279658360b13Sdlg cookie_checker_deinit(&sc->sc_cookie); 279758360b13Sdlg ret_01: 279858360b13Sdlg free(sc, M_DEVBUF, sizeof(*sc)); 279958360b13Sdlg ret_00: 280058360b13Sdlg return ENOBUFS; 280158360b13Sdlg } 280258360b13Sdlg int 280358360b13Sdlg wg_clone_destroy(struct ifnet *ifp) 280458360b13Sdlg { 280558360b13Sdlg struct wg_softc *sc = ifp->if_softc; 280658360b13Sdlg struct wg_peer *peer, *tpeer; 280758360b13Sdlg 280858360b13Sdlg KERNEL_ASSERT_LOCKED(); 280958360b13Sdlg 281058360b13Sdlg rw_enter_write(&sc->sc_lock); 281158360b13Sdlg TAILQ_FOREACH_SAFE(peer, &sc->sc_peer_seq, p_seq_entry, tpeer) 281258360b13Sdlg wg_peer_destroy(peer); 281358360b13Sdlg rw_exit_write(&sc->sc_lock); 281458360b13Sdlg 281558360b13Sdlg wg_unbind(sc); 281658360b13Sdlg if_detach(ifp); 281758360b13Sdlg 281858360b13Sdlg wg_counter--; 281958360b13Sdlg if (wg_counter == 0) { 282058360b13Sdlg KASSERT(wg_handshake_taskq != NULL && wg_crypt_taskq != NULL); 282158360b13Sdlg taskq_destroy(wg_handshake_taskq); 282258360b13Sdlg taskq_destroy(wg_crypt_taskq); 282358360b13Sdlg wg_handshake_taskq = NULL; 282458360b13Sdlg wg_crypt_taskq = NULL; 282558360b13Sdlg } 282658360b13Sdlg 2827*f08653c5Smvs WGPRINTF(LOG_INFO, sc, NULL, "Interface destroyed\n"); 282858360b13Sdlg 282958360b13Sdlg hashfree(sc->sc_index, HASHTABLE_INDEX_SIZE, M_DEVBUF); 283058360b13Sdlg hashfree(sc->sc_peer, HASHTABLE_PEER_SIZE, M_DEVBUF); 283158360b13Sdlg #ifdef INET6 283258360b13Sdlg free(sc->sc_aip6, M_RTABLE, sizeof(*sc->sc_aip6)); 283358360b13Sdlg #endif 283458360b13Sdlg free(sc->sc_aip4, M_RTABLE, sizeof(*sc->sc_aip4)); 283558360b13Sdlg cookie_checker_deinit(&sc->sc_cookie); 283658360b13Sdlg free(sc, M_DEVBUF, sizeof(*sc)); 283758360b13Sdlg return 0; 283858360b13Sdlg } 283958360b13Sdlg 284058360b13Sdlg void 284158360b13Sdlg wgattach(int nwg) 284258360b13Sdlg { 284358360b13Sdlg #ifdef WGTEST 284458360b13Sdlg cookie_test(); 284558360b13Sdlg noise_test(); 284658360b13Sdlg #endif 284758360b13Sdlg if_clone_attach(&wg_cloner); 284858360b13Sdlg 284958360b13Sdlg pool_init(&wg_aip_pool, sizeof(struct wg_aip), 0, 285058360b13Sdlg IPL_NET, 0, "wgaip", NULL); 285158360b13Sdlg pool_init(&wg_peer_pool, sizeof(struct wg_peer), 0, 285258360b13Sdlg IPL_NET, 0, "wgpeer", NULL); 285358360b13Sdlg pool_init(&wg_ratelimit_pool, sizeof(struct ratelimit_entry), 0, 285458360b13Sdlg IPL_NET, 0, "wgratelimit", NULL); 285558360b13Sdlg } 2856