xref: /openbsd-src/sys/net/if_wg.c (revision f08653c5675c062059a3f1eaa00008c5edfe93bf)
1*f08653c5Smvs /*	$OpenBSD: if_wg.c,v 1.40 2025/01/25 14:51:34 mvs Exp $ */
23ea0ed4dSjasper 
358360b13Sdlg /*
458360b13Sdlg  * Copyright (C) 2015-2020 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
558360b13Sdlg  * Copyright (C) 2019-2020 Matt Dunwoodie <ncon@noconroy.net>
658360b13Sdlg  *
758360b13Sdlg  * Permission to use, copy, modify, and distribute this software for any
858360b13Sdlg  * purpose with or without fee is hereby granted, provided that the above
958360b13Sdlg  * copyright notice and this permission notice appear in all copies.
1058360b13Sdlg  *
1158360b13Sdlg  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
1258360b13Sdlg  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
1358360b13Sdlg  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
1458360b13Sdlg  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
1558360b13Sdlg  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
1658360b13Sdlg  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
1758360b13Sdlg  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
1858360b13Sdlg  */
1958360b13Sdlg 
2058360b13Sdlg #include "bpfilter.h"
21e3884aa0Stb #include "pf.h"
2258360b13Sdlg 
2358360b13Sdlg #include <sys/types.h>
2458360b13Sdlg #include <sys/systm.h>
2558360b13Sdlg #include <sys/param.h>
2658360b13Sdlg #include <sys/pool.h>
2758360b13Sdlg 
2858360b13Sdlg #include <sys/socket.h>
2958360b13Sdlg #include <sys/socketvar.h>
3058360b13Sdlg #include <sys/percpu.h>
3158360b13Sdlg #include <sys/ioctl.h>
3258360b13Sdlg #include <sys/mbuf.h>
33*f08653c5Smvs #include <sys/syslog.h>
3458360b13Sdlg 
3558360b13Sdlg #include <net/if.h>
3658360b13Sdlg #include <net/if_var.h>
3758360b13Sdlg #include <net/if_types.h>
3858360b13Sdlg #include <net/if_wg.h>
3958360b13Sdlg 
4058360b13Sdlg #include <net/wg_noise.h>
4158360b13Sdlg #include <net/wg_cookie.h>
4258360b13Sdlg 
4358360b13Sdlg #include <net/pfvar.h>
4458360b13Sdlg #include <net/route.h>
4558360b13Sdlg #include <net/bpf.h>
4691defc8bSclaudio #include <net/art.h>
4758360b13Sdlg 
4858360b13Sdlg #include <netinet/ip.h>
4958360b13Sdlg #include <netinet/ip6.h>
5058360b13Sdlg #include <netinet/udp.h>
5158360b13Sdlg #include <netinet/in_pcb.h>
5258360b13Sdlg 
5358360b13Sdlg #include <crypto/siphash.h>
5458360b13Sdlg 
5558360b13Sdlg #define DEFAULT_MTU		1420
5658360b13Sdlg 
5758360b13Sdlg #define MAX_STAGED_PKT		128
5858360b13Sdlg #define MAX_QUEUED_PKT		1024
5958360b13Sdlg #define MAX_QUEUED_PKT_MASK	(MAX_QUEUED_PKT - 1)
6058360b13Sdlg 
6158360b13Sdlg #define MAX_QUEUED_HANDSHAKES	4096
6258360b13Sdlg 
6358360b13Sdlg #define HASHTABLE_PEER_SIZE	(1 << 11)
6458360b13Sdlg #define HASHTABLE_INDEX_SIZE	(1 << 13)
6558360b13Sdlg #define MAX_PEERS_PER_IFACE	(1 << 20)
6658360b13Sdlg 
6758360b13Sdlg #define REKEY_TIMEOUT		5
6858360b13Sdlg #define REKEY_TIMEOUT_JITTER	334 /* 1/3 sec, round for arc4random_uniform */
6958360b13Sdlg #define KEEPALIVE_TIMEOUT	10
7058360b13Sdlg #define MAX_TIMER_HANDSHAKES	(90 / REKEY_TIMEOUT)
7158360b13Sdlg #define NEW_HANDSHAKE_TIMEOUT	(REKEY_TIMEOUT + KEEPALIVE_TIMEOUT)
7258360b13Sdlg #define UNDERLOAD_TIMEOUT	1
7358360b13Sdlg 
74*f08653c5Smvs #define WGPRINTF(loglevel, sc, mtx, fmt, ...) do {		\
75*f08653c5Smvs 	if (ISSET((sc)->sc_if.if_flags, IFF_DEBUG)) {		\
76*f08653c5Smvs 		if (mtx)					\
77*f08653c5Smvs 			mtx_enter(mtx);				\
78*f08653c5Smvs 		log(loglevel, "%s: " fmt, (sc)->sc_if.if_xname, \
79*f08653c5Smvs 		    ##__VA_ARGS__);				\
80*f08653c5Smvs 		if (mtx)					\
81*f08653c5Smvs 			mtx_leave(mtx);				\
82*f08653c5Smvs 	}							\
83*f08653c5Smvs } while (0)
8458360b13Sdlg 
8558360b13Sdlg #define CONTAINER_OF(ptr, type, member) ({			\
8658360b13Sdlg 	const __typeof( ((type *)0)->member ) *__mptr = (ptr);	\
8758360b13Sdlg 	(type *)( (char *)__mptr - offsetof(type,member) );})
8858360b13Sdlg 
8958360b13Sdlg /* First byte indicating packet type on the wire */
9058360b13Sdlg #define WG_PKT_INITIATION htole32(1)
9158360b13Sdlg #define WG_PKT_RESPONSE htole32(2)
9258360b13Sdlg #define WG_PKT_COOKIE htole32(3)
9358360b13Sdlg #define WG_PKT_DATA htole32(4)
9458360b13Sdlg 
9558360b13Sdlg #define WG_PKT_WITH_PADDING(n)	(((n) + (16-1)) & (~(16-1)))
9658360b13Sdlg #define WG_KEY_SIZE		WG_KEY_LEN
9758360b13Sdlg 
9858360b13Sdlg struct wg_pkt_initiation {
9958360b13Sdlg 	uint32_t		t;
10058360b13Sdlg 	uint32_t		s_idx;
10158360b13Sdlg 	uint8_t			ue[NOISE_PUBLIC_KEY_LEN];
10258360b13Sdlg 	uint8_t			es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN];
10358360b13Sdlg 	uint8_t			ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN];
10458360b13Sdlg 	struct cookie_macs	m;
10558360b13Sdlg };
10658360b13Sdlg 
10758360b13Sdlg struct wg_pkt_response {
10858360b13Sdlg 	uint32_t		t;
10958360b13Sdlg 	uint32_t		s_idx;
11058360b13Sdlg 	uint32_t		r_idx;
11158360b13Sdlg 	uint8_t			ue[NOISE_PUBLIC_KEY_LEN];
11258360b13Sdlg 	uint8_t			en[0 + NOISE_AUTHTAG_LEN];
11358360b13Sdlg 	struct cookie_macs	m;
11458360b13Sdlg };
11558360b13Sdlg 
11658360b13Sdlg struct wg_pkt_cookie {
11758360b13Sdlg 	uint32_t		t;
11858360b13Sdlg 	uint32_t		r_idx;
11958360b13Sdlg 	uint8_t			nonce[COOKIE_NONCE_SIZE];
12058360b13Sdlg 	uint8_t			ec[COOKIE_ENCRYPTED_SIZE];
12158360b13Sdlg };
12258360b13Sdlg 
12358360b13Sdlg struct wg_pkt_data {
12458360b13Sdlg 	uint32_t		t;
12558360b13Sdlg 	uint32_t		r_idx;
12658360b13Sdlg 	uint8_t			nonce[sizeof(uint64_t)];
12758360b13Sdlg 	uint8_t			buf[];
12858360b13Sdlg };
12958360b13Sdlg 
13058360b13Sdlg struct wg_endpoint {
13158360b13Sdlg 	union {
13258360b13Sdlg 		struct sockaddr		r_sa;
13358360b13Sdlg 		struct sockaddr_in	r_sin;
13458360b13Sdlg #ifdef INET6
13558360b13Sdlg 		struct sockaddr_in6	r_sin6;
13658360b13Sdlg #endif
13758360b13Sdlg 	} e_remote;
13858360b13Sdlg 	union {
13958360b13Sdlg 		struct in_addr		l_in;
14058360b13Sdlg #ifdef INET6
14158360b13Sdlg 		struct in6_pktinfo	l_pktinfo6;
14258360b13Sdlg #define l_in6 l_pktinfo6.ipi6_addr
14358360b13Sdlg #endif
14458360b13Sdlg 	} e_local;
14558360b13Sdlg };
14658360b13Sdlg 
14758360b13Sdlg struct wg_tag {
14858360b13Sdlg 	struct wg_endpoint	 t_endpoint;
14958360b13Sdlg 	struct wg_peer		*t_peer;
15058360b13Sdlg 	struct mbuf		*t_mbuf;
15158360b13Sdlg 	int			 t_done;
15258360b13Sdlg 	int			 t_mtu;
15358360b13Sdlg };
15458360b13Sdlg 
15558360b13Sdlg struct wg_index {
15658360b13Sdlg 	LIST_ENTRY(wg_index)	 i_entry;
15758360b13Sdlg 	SLIST_ENTRY(wg_index)	 i_unused_entry;
15858360b13Sdlg 	uint32_t		 i_key;
15958360b13Sdlg 	struct noise_remote	*i_value;
16058360b13Sdlg };
16158360b13Sdlg 
16258360b13Sdlg struct wg_timers {
163efbb2e09Smvs 	/* t_mtx is for blocking wg_timers_event_* when setting t_disabled. */
164efbb2e09Smvs 	struct mutex		 t_mtx;
16558360b13Sdlg 
16658360b13Sdlg 	int			 t_disabled;
16758360b13Sdlg 	int			 t_need_another_keepalive;
16858360b13Sdlg 	uint16_t		 t_persistent_keepalive_interval;
16958360b13Sdlg 	struct timeout		 t_new_handshake;
17058360b13Sdlg 	struct timeout		 t_send_keepalive;
17158360b13Sdlg 	struct timeout		 t_retry_handshake;
17258360b13Sdlg 	struct timeout		 t_zero_key_material;
17358360b13Sdlg 	struct timeout		 t_persistent_keepalive;
17458360b13Sdlg 
17558360b13Sdlg 	struct mutex		 t_handshake_mtx;
17658360b13Sdlg 	struct timespec		 t_handshake_last_sent;	/* nanouptime */
17758360b13Sdlg 	struct timespec		 t_handshake_complete;	/* nanotime */
17858360b13Sdlg 	int			 t_handshake_retries;
17958360b13Sdlg };
18058360b13Sdlg 
18158360b13Sdlg struct wg_aip {
18258360b13Sdlg 	struct art_node		 a_node;
18358360b13Sdlg 	LIST_ENTRY(wg_aip)	 a_entry;
18458360b13Sdlg 	struct wg_peer		*a_peer;
18558360b13Sdlg 	struct wg_aip_io	 a_data;
18658360b13Sdlg };
18758360b13Sdlg 
18858360b13Sdlg struct wg_queue {
18958360b13Sdlg 	struct mutex		 q_mtx;
19058360b13Sdlg 	struct mbuf_list	 q_list;
19158360b13Sdlg };
19258360b13Sdlg 
19358360b13Sdlg struct wg_ring {
19458360b13Sdlg 	struct mutex	 r_mtx;
19558360b13Sdlg 	uint32_t	 r_head;
19658360b13Sdlg 	uint32_t	 r_tail;
19758360b13Sdlg 	struct mbuf	*r_buf[MAX_QUEUED_PKT];
19858360b13Sdlg };
19958360b13Sdlg 
20058360b13Sdlg struct wg_peer {
20158360b13Sdlg 	LIST_ENTRY(wg_peer)	 p_pubkey_entry;
20258360b13Sdlg 	TAILQ_ENTRY(wg_peer)	 p_seq_entry;
20358360b13Sdlg 	uint64_t		 p_id;
20458360b13Sdlg 	struct wg_softc		*p_sc;
20558360b13Sdlg 
20658360b13Sdlg 	struct noise_remote	 p_remote;
20758360b13Sdlg 	struct cookie_maker	 p_cookie;
20858360b13Sdlg 	struct wg_timers	 p_timers;
20958360b13Sdlg 
21058360b13Sdlg 	struct mutex		 p_counters_mtx;
21158360b13Sdlg 	uint64_t		 p_counters_tx;
21258360b13Sdlg 	uint64_t		 p_counters_rx;
21358360b13Sdlg 
21458360b13Sdlg 	struct mutex		 p_endpoint_mtx;
21558360b13Sdlg 	struct wg_endpoint	 p_endpoint;
21658360b13Sdlg 
21758360b13Sdlg 	struct task		 p_send_initiation;
21858360b13Sdlg 	struct task		 p_send_keepalive;
21958360b13Sdlg 	struct task		 p_clear_secrets;
22058360b13Sdlg 	struct task		 p_deliver_out;
22158360b13Sdlg 	struct task		 p_deliver_in;
22258360b13Sdlg 
22358360b13Sdlg 	struct mbuf_queue	 p_stage_queue;
22458360b13Sdlg 	struct wg_queue		 p_encap_queue;
22558360b13Sdlg 	struct wg_queue		 p_decap_queue;
22658360b13Sdlg 
22758360b13Sdlg 	SLIST_HEAD(,wg_index)	 p_unused_index;
22858360b13Sdlg 	struct wg_index		 p_index[3];
22958360b13Sdlg 
23058360b13Sdlg 	LIST_HEAD(,wg_aip)	 p_aip;
23158360b13Sdlg 
23258360b13Sdlg 	SLIST_ENTRY(wg_peer)	 p_start_list;
23358360b13Sdlg 	int			 p_start_onlist;
23411d9cfa5Skn 
23511d9cfa5Skn 	char			 p_description[IFDESCRSIZE];
23658360b13Sdlg };
23758360b13Sdlg 
23858360b13Sdlg struct wg_softc {
23958360b13Sdlg 	struct ifnet		 sc_if;
24058360b13Sdlg 	SIPHASH_KEY		 sc_secret;
24158360b13Sdlg 
24258360b13Sdlg 	struct rwlock		 sc_lock;
24358360b13Sdlg 	struct noise_local	 sc_local;
24458360b13Sdlg 	struct cookie_checker	 sc_cookie;
24558360b13Sdlg 	in_port_t		 sc_udp_port;
24658360b13Sdlg 	int			 sc_udp_rtable;
24758360b13Sdlg 
24858360b13Sdlg 	struct rwlock		 sc_so_lock;
24958360b13Sdlg 	struct socket		*sc_so4;
25058360b13Sdlg #ifdef INET6
25158360b13Sdlg 	struct socket		*sc_so6;
25258360b13Sdlg #endif
25358360b13Sdlg 
25458360b13Sdlg 	size_t			 sc_aip_num;
25558360b13Sdlg 	struct art_root		*sc_aip4;
25658360b13Sdlg #ifdef INET6
25758360b13Sdlg 	struct art_root		*sc_aip6;
25858360b13Sdlg #endif
25958360b13Sdlg 
26058360b13Sdlg 	struct rwlock		 sc_peer_lock;
26158360b13Sdlg 	size_t			 sc_peer_num;
26258360b13Sdlg 	LIST_HEAD(,wg_peer)	*sc_peer;
26358360b13Sdlg 	TAILQ_HEAD(,wg_peer)	 sc_peer_seq;
26458360b13Sdlg 	u_long			 sc_peer_mask;
26558360b13Sdlg 
26658360b13Sdlg 	struct mutex		 sc_index_mtx;
26758360b13Sdlg 	LIST_HEAD(,wg_index)	*sc_index;
26858360b13Sdlg 	u_long			 sc_index_mask;
26958360b13Sdlg 
27058360b13Sdlg 	struct task		 sc_handshake;
27158360b13Sdlg 	struct mbuf_queue	 sc_handshake_queue;
27258360b13Sdlg 
27358360b13Sdlg 	struct task		 sc_encap;
27458360b13Sdlg 	struct task		 sc_decap;
27558360b13Sdlg 	struct wg_ring		 sc_encap_ring;
27658360b13Sdlg 	struct wg_ring		 sc_decap_ring;
27758360b13Sdlg };
27858360b13Sdlg 
27958360b13Sdlg struct wg_peer *
28058360b13Sdlg 	wg_peer_create(struct wg_softc *, uint8_t[WG_KEY_SIZE]);
28158360b13Sdlg struct wg_peer *
28258360b13Sdlg 	wg_peer_lookup(struct wg_softc *, const uint8_t[WG_KEY_SIZE]);
28358360b13Sdlg void	wg_peer_destroy(struct wg_peer *);
28458360b13Sdlg void	wg_peer_set_endpoint_from_tag(struct wg_peer *, struct wg_tag *);
28558360b13Sdlg void	wg_peer_set_sockaddr(struct wg_peer *, struct sockaddr *);
28658360b13Sdlg int	wg_peer_get_sockaddr(struct wg_peer *, struct sockaddr *);
28758360b13Sdlg void	wg_peer_clear_src(struct wg_peer *);
28858360b13Sdlg void	wg_peer_get_endpoint(struct wg_peer *, struct wg_endpoint *);
28958360b13Sdlg void	wg_peer_counters_add(struct wg_peer *, uint64_t, uint64_t);
29058360b13Sdlg 
29158360b13Sdlg int	wg_aip_add(struct wg_softc *, struct wg_peer *, struct wg_aip_io *);
29258360b13Sdlg struct wg_peer *
29358360b13Sdlg 	wg_aip_lookup(struct art_root *, void *);
29458360b13Sdlg int	wg_aip_remove(struct wg_softc *, struct wg_peer *,
29558360b13Sdlg 	    struct wg_aip_io *);
29658360b13Sdlg 
29758360b13Sdlg int	wg_socket_open(struct socket **, int, in_port_t *, int *, void *);
29858360b13Sdlg void	wg_socket_close(struct socket **);
29958360b13Sdlg int	wg_bind(struct wg_softc *, in_port_t *, int *);
30058360b13Sdlg void	wg_unbind(struct wg_softc *);
30158360b13Sdlg int	wg_send(struct wg_softc *, struct wg_endpoint *, struct mbuf *);
30258360b13Sdlg void	wg_send_buf(struct wg_softc *, struct wg_endpoint *, uint8_t *,
30358360b13Sdlg 	    size_t);
30458360b13Sdlg 
30558360b13Sdlg struct wg_tag *
30658360b13Sdlg 	wg_tag_get(struct mbuf *);
30758360b13Sdlg 
30858360b13Sdlg void	wg_timers_init(struct wg_timers *);
30958360b13Sdlg void	wg_timers_enable(struct wg_timers *);
31058360b13Sdlg void	wg_timers_disable(struct wg_timers *);
31158360b13Sdlg void	wg_timers_set_persistent_keepalive(struct wg_timers *, uint16_t);
31258360b13Sdlg int	wg_timers_get_persistent_keepalive(struct wg_timers *, uint16_t *);
31358360b13Sdlg void	wg_timers_get_last_handshake(struct wg_timers *, struct timespec *);
31458360b13Sdlg int	wg_timers_expired_handshake_last_sent(struct wg_timers *);
31558360b13Sdlg int	wg_timers_check_handshake_last_sent(struct wg_timers *);
31658360b13Sdlg 
31758360b13Sdlg void	wg_timers_event_data_sent(struct wg_timers *);
31858360b13Sdlg void	wg_timers_event_data_received(struct wg_timers *);
31958360b13Sdlg void	wg_timers_event_any_authenticated_packet_sent(struct wg_timers *);
32058360b13Sdlg void	wg_timers_event_any_authenticated_packet_received(struct wg_timers *);
32158360b13Sdlg void	wg_timers_event_handshake_initiated(struct wg_timers *);
32258360b13Sdlg void	wg_timers_event_handshake_responded(struct wg_timers *);
32358360b13Sdlg void	wg_timers_event_handshake_complete(struct wg_timers *);
32458360b13Sdlg void	wg_timers_event_session_derived(struct wg_timers *);
32558360b13Sdlg void	wg_timers_event_any_authenticated_packet_traversal(struct wg_timers *);
32658360b13Sdlg void	wg_timers_event_want_initiation(struct wg_timers *);
32758360b13Sdlg void	wg_timers_event_reset_handshake_last_sent(struct wg_timers *);
32858360b13Sdlg 
32958360b13Sdlg void	wg_timers_run_send_initiation(void *, int);
33058360b13Sdlg void	wg_timers_run_retry_handshake(void *);
33158360b13Sdlg void	wg_timers_run_send_keepalive(void *);
33258360b13Sdlg void	wg_timers_run_new_handshake(void *);
33358360b13Sdlg void	wg_timers_run_zero_key_material(void *);
33458360b13Sdlg void	wg_timers_run_persistent_keepalive(void *);
33558360b13Sdlg 
33658360b13Sdlg void	wg_peer_send_buf(struct wg_peer *, uint8_t *, size_t);
33758360b13Sdlg void	wg_send_initiation(void *);
33858360b13Sdlg void	wg_send_response(struct wg_peer *);
33958360b13Sdlg void	wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t,
34094dfe323Stb 	    struct wg_endpoint *);
34158360b13Sdlg void	wg_send_keepalive(void *);
34258360b13Sdlg void	wg_peer_clear_secrets(void *);
34358360b13Sdlg void	wg_handshake(struct wg_softc *, struct mbuf *);
34458360b13Sdlg void	wg_handshake_worker(void *);
34558360b13Sdlg 
34658360b13Sdlg void	wg_encap(struct wg_softc *, struct mbuf *);
34758360b13Sdlg void	wg_decap(struct wg_softc *, struct mbuf *);
34858360b13Sdlg void	wg_encap_worker(void *);
34958360b13Sdlg void	wg_decap_worker(void *);
35058360b13Sdlg void	wg_deliver_out(void *);
35158360b13Sdlg void	wg_deliver_in(void *);
35258360b13Sdlg 
35358360b13Sdlg int	wg_queue_in(struct wg_softc *, struct wg_peer *, struct mbuf *);
35458360b13Sdlg void	wg_queue_out(struct wg_softc *, struct wg_peer *);
35558360b13Sdlg struct mbuf *
35658360b13Sdlg 	wg_ring_dequeue(struct wg_ring *);
35758360b13Sdlg struct mbuf *
35858360b13Sdlg 	wg_queue_dequeue(struct wg_queue *, struct wg_tag **);
35958360b13Sdlg size_t	wg_queue_len(struct wg_queue *);
36058360b13Sdlg 
36158360b13Sdlg struct noise_remote *
36258360b13Sdlg 	wg_remote_get(void *, uint8_t[NOISE_PUBLIC_KEY_LEN]);
36358360b13Sdlg uint32_t
36458360b13Sdlg 	wg_index_set(void *, struct noise_remote *);
36558360b13Sdlg struct noise_remote *
36658360b13Sdlg 	wg_index_get(void *, uint32_t);
36758360b13Sdlg void	wg_index_drop(void *, uint32_t);
36858360b13Sdlg 
36958360b13Sdlg struct mbuf *
37058360b13Sdlg 	wg_input(void *, struct mbuf *, struct ip *, struct ip6_hdr *, void *,
37158360b13Sdlg 	    int);
37258360b13Sdlg int	wg_output(struct ifnet *, struct mbuf *, struct sockaddr *,
37358360b13Sdlg 	    struct rtentry *);
37458360b13Sdlg int	wg_ioctl_set(struct wg_softc *, struct wg_data_io *);
37558360b13Sdlg int	wg_ioctl_get(struct wg_softc *, struct wg_data_io *);
37658360b13Sdlg int	wg_ioctl(struct ifnet *, u_long, caddr_t);
37758360b13Sdlg int	wg_up(struct wg_softc *);
37858360b13Sdlg void	wg_down(struct wg_softc *);
37958360b13Sdlg 
38058360b13Sdlg int	wg_clone_create(struct if_clone *, int);
38158360b13Sdlg int	wg_clone_destroy(struct ifnet *);
38258360b13Sdlg void	wgattach(int);
38358360b13Sdlg 
38458360b13Sdlg uint64_t	peer_counter = 0;
38558360b13Sdlg struct pool	wg_aip_pool;
38658360b13Sdlg struct pool	wg_peer_pool;
38758360b13Sdlg struct pool	wg_ratelimit_pool;
38858360b13Sdlg struct timeval	underload_interval = { UNDERLOAD_TIMEOUT, 0 };
38958360b13Sdlg 
39058360b13Sdlg size_t		 wg_counter = 0;
39158360b13Sdlg struct taskq	*wg_handshake_taskq;
39258360b13Sdlg struct taskq	*wg_crypt_taskq;
39358360b13Sdlg 
39458360b13Sdlg struct if_clone	wg_cloner =
39558360b13Sdlg     IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy);
39658360b13Sdlg 
39758360b13Sdlg struct wg_peer *
39858360b13Sdlg wg_peer_create(struct wg_softc *sc, uint8_t public[WG_KEY_SIZE])
39958360b13Sdlg {
40058360b13Sdlg 	struct wg_peer	*peer;
40158360b13Sdlg 	uint64_t	 idx;
40258360b13Sdlg 
40358360b13Sdlg 	rw_assert_wrlock(&sc->sc_lock);
40458360b13Sdlg 
40558360b13Sdlg 	if (sc->sc_peer_num >= MAX_PEERS_PER_IFACE)
40658360b13Sdlg 		return NULL;
40758360b13Sdlg 
40858360b13Sdlg 	if ((peer = pool_get(&wg_peer_pool, PR_NOWAIT)) == NULL)
40958360b13Sdlg 		return NULL;
41058360b13Sdlg 
41158360b13Sdlg 	peer->p_id = peer_counter++;
41258360b13Sdlg 	peer->p_sc = sc;
41358360b13Sdlg 
41458360b13Sdlg 	noise_remote_init(&peer->p_remote, public, &sc->sc_local);
41558360b13Sdlg 	cookie_maker_init(&peer->p_cookie, public);
41658360b13Sdlg 	wg_timers_init(&peer->p_timers);
41758360b13Sdlg 
41858360b13Sdlg 	mtx_init(&peer->p_counters_mtx, IPL_NET);
41958360b13Sdlg 	peer->p_counters_tx = 0;
42058360b13Sdlg 	peer->p_counters_rx = 0;
42158360b13Sdlg 
42211d9cfa5Skn 	strlcpy(peer->p_description, "", IFDESCRSIZE);
42311d9cfa5Skn 
42458360b13Sdlg 	mtx_init(&peer->p_endpoint_mtx, IPL_NET);
42558360b13Sdlg 	bzero(&peer->p_endpoint, sizeof(peer->p_endpoint));
42658360b13Sdlg 
42758360b13Sdlg 	task_set(&peer->p_send_initiation, wg_send_initiation, peer);
42858360b13Sdlg 	task_set(&peer->p_send_keepalive, wg_send_keepalive, peer);
42958360b13Sdlg 	task_set(&peer->p_clear_secrets, wg_peer_clear_secrets, peer);
43058360b13Sdlg 	task_set(&peer->p_deliver_out, wg_deliver_out, peer);
43158360b13Sdlg 	task_set(&peer->p_deliver_in, wg_deliver_in, peer);
43258360b13Sdlg 
43358360b13Sdlg 	mq_init(&peer->p_stage_queue, MAX_STAGED_PKT, IPL_NET);
43458360b13Sdlg 	mtx_init(&peer->p_encap_queue.q_mtx, IPL_NET);
43558360b13Sdlg 	ml_init(&peer->p_encap_queue.q_list);
43658360b13Sdlg 	mtx_init(&peer->p_decap_queue.q_mtx, IPL_NET);
43758360b13Sdlg 	ml_init(&peer->p_decap_queue.q_list);
43858360b13Sdlg 
43958360b13Sdlg 	SLIST_INIT(&peer->p_unused_index);
44058360b13Sdlg 	SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[0],
44158360b13Sdlg 	    i_unused_entry);
44258360b13Sdlg 	SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[1],
44358360b13Sdlg 	    i_unused_entry);
44458360b13Sdlg 	SLIST_INSERT_HEAD(&peer->p_unused_index, &peer->p_index[2],
44558360b13Sdlg 	    i_unused_entry);
44658360b13Sdlg 
44758360b13Sdlg 	LIST_INIT(&peer->p_aip);
44858360b13Sdlg 
44958360b13Sdlg 	peer->p_start_onlist = 0;
45058360b13Sdlg 
45158360b13Sdlg 	idx = SipHash24(&sc->sc_secret, public, WG_KEY_SIZE);
45258360b13Sdlg 	idx &= sc->sc_peer_mask;
45358360b13Sdlg 
45458360b13Sdlg 	rw_enter_write(&sc->sc_peer_lock);
45558360b13Sdlg 	LIST_INSERT_HEAD(&sc->sc_peer[idx], peer, p_pubkey_entry);
45658360b13Sdlg 	TAILQ_INSERT_TAIL(&sc->sc_peer_seq, peer, p_seq_entry);
45758360b13Sdlg 	sc->sc_peer_num++;
45858360b13Sdlg 	rw_exit_write(&sc->sc_peer_lock);
45958360b13Sdlg 
460*f08653c5Smvs 	WGPRINTF(LOG_INFO, sc, NULL, "Peer %llu created\n", peer->p_id);
46158360b13Sdlg 	return peer;
46258360b13Sdlg }
46358360b13Sdlg 
46458360b13Sdlg struct wg_peer *
46558360b13Sdlg wg_peer_lookup(struct wg_softc *sc, const uint8_t public[WG_KEY_SIZE])
46658360b13Sdlg {
46758360b13Sdlg 	uint8_t		 peer_key[WG_KEY_SIZE];
46858360b13Sdlg 	struct wg_peer	*peer;
46958360b13Sdlg 	uint64_t	 idx;
47058360b13Sdlg 
47158360b13Sdlg 	idx = SipHash24(&sc->sc_secret, public, WG_KEY_SIZE);
47258360b13Sdlg 	idx &= sc->sc_peer_mask;
47358360b13Sdlg 
47458360b13Sdlg 	rw_enter_read(&sc->sc_peer_lock);
47558360b13Sdlg 	LIST_FOREACH(peer, &sc->sc_peer[idx], p_pubkey_entry) {
47658360b13Sdlg 		noise_remote_keys(&peer->p_remote, peer_key, NULL);
47758360b13Sdlg 		if (timingsafe_bcmp(peer_key, public, WG_KEY_SIZE) == 0)
47858360b13Sdlg 			goto done;
47958360b13Sdlg 	}
48058360b13Sdlg 	peer = NULL;
48158360b13Sdlg done:
48258360b13Sdlg 	rw_exit_read(&sc->sc_peer_lock);
48358360b13Sdlg 	return peer;
48458360b13Sdlg }
48558360b13Sdlg 
48658360b13Sdlg void
48758360b13Sdlg wg_peer_destroy(struct wg_peer *peer)
48858360b13Sdlg {
48958360b13Sdlg 	struct wg_softc	*sc = peer->p_sc;
49058360b13Sdlg 	struct wg_aip *aip, *taip;
49158360b13Sdlg 
49258360b13Sdlg 	rw_assert_wrlock(&sc->sc_lock);
49358360b13Sdlg 
49458360b13Sdlg 	/*
49558360b13Sdlg 	 * Remove peer from the pubkey hashtable and disable all timeouts.
49658360b13Sdlg 	 * After this, and flushing wg_handshake_taskq, then no more handshakes
49758360b13Sdlg 	 * can be started.
49858360b13Sdlg 	 */
49958360b13Sdlg 	rw_enter_write(&sc->sc_peer_lock);
50058360b13Sdlg 	LIST_REMOVE(peer, p_pubkey_entry);
50158360b13Sdlg 	TAILQ_REMOVE(&sc->sc_peer_seq, peer, p_seq_entry);
50258360b13Sdlg 	sc->sc_peer_num--;
50358360b13Sdlg 	rw_exit_write(&sc->sc_peer_lock);
50458360b13Sdlg 
50558360b13Sdlg 	wg_timers_disable(&peer->p_timers);
50658360b13Sdlg 
50758360b13Sdlg 	taskq_barrier(wg_handshake_taskq);
50858360b13Sdlg 
50958360b13Sdlg 	/*
51058360b13Sdlg 	 * Now we drop all allowed ips, to drop all outgoing packets to the
51158360b13Sdlg 	 * peer. Then drop all the indexes to drop all incoming packets to the
51258360b13Sdlg 	 * peer. Then we can flush if_snd, wg_crypt_taskq and then nettq to
51358360b13Sdlg 	 * ensure no more references to the peer exist.
51458360b13Sdlg 	 */
51558360b13Sdlg 	LIST_FOREACH_SAFE(aip, &peer->p_aip, a_entry, taip)
51658360b13Sdlg 		wg_aip_remove(sc, peer, &aip->a_data);
51758360b13Sdlg 
51858360b13Sdlg 	noise_remote_clear(&peer->p_remote);
51958360b13Sdlg 
52058360b13Sdlg 	NET_LOCK();
52158360b13Sdlg 	while (!ifq_empty(&sc->sc_if.if_snd)) {
5228c2c9fccSmvs 		/*
5238c2c9fccSmvs 		 * XXX: `if_snd' of stopped interface could still
5248c2c9fccSmvs 		 * contain packets
5258c2c9fccSmvs 		 */
5268c2c9fccSmvs 		if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)) {
5278c2c9fccSmvs 			ifq_purge(&sc->sc_if.if_snd);
5288c2c9fccSmvs 			continue;
5298c2c9fccSmvs 		}
53058360b13Sdlg 		NET_UNLOCK();
5314bd216efSmvs 		tsleep_nsec(&nowake, PWAIT, "wg_ifq", 1000);
53258360b13Sdlg 		NET_LOCK();
53358360b13Sdlg 	}
53458360b13Sdlg 	NET_UNLOCK();
53558360b13Sdlg 
53658360b13Sdlg 	taskq_barrier(wg_crypt_taskq);
53758360b13Sdlg 	taskq_barrier(net_tq(sc->sc_if.if_index));
53858360b13Sdlg 
53976ac1d3dSyasuoka 	if (!mq_empty(&peer->p_stage_queue))
54076ac1d3dSyasuoka 		mq_purge(&peer->p_stage_queue);
54176ac1d3dSyasuoka 
542*f08653c5Smvs 	WGPRINTF(LOG_INFO, sc, NULL, "Peer %llu destroyed\n", peer->p_id);
54358360b13Sdlg 	explicit_bzero(peer, sizeof(*peer));
54458360b13Sdlg 	pool_put(&wg_peer_pool, peer);
54558360b13Sdlg }
54658360b13Sdlg 
54758360b13Sdlg void
54858360b13Sdlg wg_peer_set_endpoint_from_tag(struct wg_peer *peer, struct wg_tag *t)
54958360b13Sdlg {
55058360b13Sdlg 	if (memcmp(&t->t_endpoint, &peer->p_endpoint,
55158360b13Sdlg 	    sizeof(t->t_endpoint)) == 0)
55258360b13Sdlg 		return;
55358360b13Sdlg 
55458360b13Sdlg 	mtx_enter(&peer->p_endpoint_mtx);
55558360b13Sdlg 	peer->p_endpoint = t->t_endpoint;
55658360b13Sdlg 	mtx_leave(&peer->p_endpoint_mtx);
55758360b13Sdlg }
55858360b13Sdlg 
55958360b13Sdlg void
56058360b13Sdlg wg_peer_set_sockaddr(struct wg_peer *peer, struct sockaddr *remote)
56158360b13Sdlg {
56258360b13Sdlg 	mtx_enter(&peer->p_endpoint_mtx);
56358360b13Sdlg 	memcpy(&peer->p_endpoint.e_remote, remote,
56458360b13Sdlg 	       sizeof(peer->p_endpoint.e_remote));
56558360b13Sdlg 	bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local));
56658360b13Sdlg 	mtx_leave(&peer->p_endpoint_mtx);
56758360b13Sdlg }
56858360b13Sdlg 
56958360b13Sdlg int
57058360b13Sdlg wg_peer_get_sockaddr(struct wg_peer *peer, struct sockaddr *remote)
57158360b13Sdlg {
57258360b13Sdlg 	int	ret = 0;
57358360b13Sdlg 
57458360b13Sdlg 	mtx_enter(&peer->p_endpoint_mtx);
57558360b13Sdlg 	if (peer->p_endpoint.e_remote.r_sa.sa_family != AF_UNSPEC)
57658360b13Sdlg 		memcpy(remote, &peer->p_endpoint.e_remote,
57758360b13Sdlg 		       sizeof(peer->p_endpoint.e_remote));
57858360b13Sdlg 	else
57958360b13Sdlg 		ret = ENOENT;
58058360b13Sdlg 	mtx_leave(&peer->p_endpoint_mtx);
58158360b13Sdlg 	return ret;
58258360b13Sdlg }
58358360b13Sdlg 
58458360b13Sdlg void
58558360b13Sdlg wg_peer_clear_src(struct wg_peer *peer)
58658360b13Sdlg {
58758360b13Sdlg 	mtx_enter(&peer->p_endpoint_mtx);
58858360b13Sdlg 	bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local));
58958360b13Sdlg 	mtx_leave(&peer->p_endpoint_mtx);
59058360b13Sdlg }
59158360b13Sdlg 
59258360b13Sdlg void
59358360b13Sdlg wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *endpoint)
59458360b13Sdlg {
59558360b13Sdlg 	mtx_enter(&peer->p_endpoint_mtx);
59658360b13Sdlg 	memcpy(endpoint, &peer->p_endpoint, sizeof(*endpoint));
59758360b13Sdlg 	mtx_leave(&peer->p_endpoint_mtx);
59858360b13Sdlg }
59958360b13Sdlg 
60058360b13Sdlg void
60158360b13Sdlg wg_peer_counters_add(struct wg_peer *peer, uint64_t tx, uint64_t rx)
60258360b13Sdlg {
60358360b13Sdlg 	mtx_enter(&peer->p_counters_mtx);
60458360b13Sdlg 	peer->p_counters_tx += tx;
60558360b13Sdlg 	peer->p_counters_rx += rx;
60658360b13Sdlg 	mtx_leave(&peer->p_counters_mtx);
60758360b13Sdlg }
60858360b13Sdlg 
60958360b13Sdlg int
61058360b13Sdlg wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, struct wg_aip_io *d)
61158360b13Sdlg {
61258360b13Sdlg 	struct art_root	*root;
61358360b13Sdlg 	struct art_node	*node;
61458360b13Sdlg 	struct wg_aip	*aip;
61558360b13Sdlg 	int		 ret = 0;
61658360b13Sdlg 
61758360b13Sdlg 	switch (d->a_af) {
61858360b13Sdlg 	case AF_INET:	root = sc->sc_aip4; break;
61958360b13Sdlg #ifdef INET6
62058360b13Sdlg 	case AF_INET6:	root = sc->sc_aip6; break;
62158360b13Sdlg #endif
62258360b13Sdlg 	default: return EAFNOSUPPORT;
62358360b13Sdlg 	}
62458360b13Sdlg 
6254023ddd7Sderaadt 	if ((aip = pool_get(&wg_aip_pool, PR_NOWAIT|PR_ZERO)) == NULL)
62658360b13Sdlg 		return ENOBUFS;
62758360b13Sdlg 
62858360b13Sdlg 	rw_enter_write(&root->ar_lock);
62958360b13Sdlg 	node = art_insert(root, &aip->a_node, &d->a_addr, d->a_cidr);
63058360b13Sdlg 
63158360b13Sdlg 	if (node == &aip->a_node) {
63258360b13Sdlg 		aip->a_peer = peer;
63358360b13Sdlg 		aip->a_data = *d;
63458360b13Sdlg 		LIST_INSERT_HEAD(&peer->p_aip, aip, a_entry);
63558360b13Sdlg 		sc->sc_aip_num++;
63658360b13Sdlg 	} else {
63758360b13Sdlg 		pool_put(&wg_aip_pool, aip);
63858360b13Sdlg 		aip = (struct wg_aip *) node;
63958360b13Sdlg 		if (aip->a_peer != peer) {
64058360b13Sdlg 			LIST_REMOVE(aip, a_entry);
64158360b13Sdlg 			LIST_INSERT_HEAD(&peer->p_aip, aip, a_entry);
64258360b13Sdlg 			aip->a_peer = peer;
64358360b13Sdlg 		}
64458360b13Sdlg 	}
64558360b13Sdlg 	rw_exit_write(&root->ar_lock);
64658360b13Sdlg 	return ret;
64758360b13Sdlg }
64858360b13Sdlg 
64958360b13Sdlg struct wg_peer *
65058360b13Sdlg wg_aip_lookup(struct art_root *root, void *addr)
65158360b13Sdlg {
65258360b13Sdlg 	struct srp_ref	 sr;
65358360b13Sdlg 	struct art_node	*node;
65458360b13Sdlg 
65558360b13Sdlg 	node = art_match(root, addr, &sr);
65658360b13Sdlg 	srp_leave(&sr);
65758360b13Sdlg 
65858360b13Sdlg 	return node == NULL ? NULL : ((struct wg_aip *) node)->a_peer;
65958360b13Sdlg }
66058360b13Sdlg 
66158360b13Sdlg int
66258360b13Sdlg wg_aip_remove(struct wg_softc *sc, struct wg_peer *peer, struct wg_aip_io *d)
66358360b13Sdlg {
66458360b13Sdlg 	struct srp_ref	 sr;
66558360b13Sdlg 	struct art_root	*root;
66658360b13Sdlg 	struct art_node	*node;
66758360b13Sdlg 	struct wg_aip	*aip;
66858360b13Sdlg 	int		 ret = 0;
66958360b13Sdlg 
67058360b13Sdlg 	switch (d->a_af) {
67158360b13Sdlg 	case AF_INET:	root = sc->sc_aip4; break;
67258360b13Sdlg #ifdef INET6
67358360b13Sdlg 	case AF_INET6:	root = sc->sc_aip6; break;
67458360b13Sdlg #endif
67558360b13Sdlg 	default: return EAFNOSUPPORT;
67658360b13Sdlg 	}
67758360b13Sdlg 
67858360b13Sdlg 	rw_enter_write(&root->ar_lock);
67958360b13Sdlg 	if ((node = art_lookup(root, &d->a_addr, d->a_cidr, &sr)) == NULL) {
68058360b13Sdlg 		ret = ENOENT;
68158360b13Sdlg 	} else if (((struct wg_aip *) node)->a_peer != peer) {
68258360b13Sdlg 		ret = EXDEV;
68358360b13Sdlg 	} else {
68458360b13Sdlg 		aip = (struct wg_aip *)node;
68558360b13Sdlg 		if (art_delete(root, node, &d->a_addr, d->a_cidr) == NULL)
68658360b13Sdlg 			panic("art_delete failed to delete node %p", node);
68758360b13Sdlg 
68858360b13Sdlg 		sc->sc_aip_num--;
68958360b13Sdlg 		LIST_REMOVE(aip, a_entry);
69058360b13Sdlg 		pool_put(&wg_aip_pool, aip);
69158360b13Sdlg 	}
69258360b13Sdlg 
69358360b13Sdlg 	srp_leave(&sr);
69458360b13Sdlg 	rw_exit_write(&root->ar_lock);
69558360b13Sdlg 	return ret;
69658360b13Sdlg }
69758360b13Sdlg 
69858360b13Sdlg int
69958360b13Sdlg wg_socket_open(struct socket **so, int af, in_port_t *port,
70058360b13Sdlg     int *rtable, void *upcall_arg)
70158360b13Sdlg {
70258360b13Sdlg 	struct mbuf		 mhostnam, mrtable;
70358360b13Sdlg #ifdef INET6
70458360b13Sdlg 	struct sockaddr_in6	*sin6;
70558360b13Sdlg #endif
70658360b13Sdlg 	struct sockaddr_in	*sin;
70784245c07Sclaudio 	int			 ret;
70858360b13Sdlg 
70958360b13Sdlg 	m_inithdr(&mhostnam);
71058360b13Sdlg 	m_inithdr(&mrtable);
71158360b13Sdlg 
71258360b13Sdlg 	bzero(mtod(&mrtable, u_int *), sizeof(u_int));
71358360b13Sdlg 	*mtod(&mrtable, u_int *) = *rtable;
71458360b13Sdlg 	mrtable.m_len = sizeof(u_int);
71558360b13Sdlg 
71658360b13Sdlg 	if (af == AF_INET) {
71758360b13Sdlg 		sin = mtod(&mhostnam, struct sockaddr_in *);
71858360b13Sdlg 		bzero(sin, sizeof(*sin));
71958360b13Sdlg 		sin->sin_len = sizeof(*sin);
72058360b13Sdlg 		sin->sin_family = AF_INET;
72158360b13Sdlg 		sin->sin_port = *port;
72258360b13Sdlg 		sin->sin_addr.s_addr = INADDR_ANY;
72358360b13Sdlg 		mhostnam.m_len = sin->sin_len;
72458360b13Sdlg #ifdef INET6
72558360b13Sdlg 	} else if (af == AF_INET6) {
72658360b13Sdlg 		sin6 = mtod(&mhostnam, struct sockaddr_in6 *);
72758360b13Sdlg 		bzero(sin6, sizeof(*sin6));
72858360b13Sdlg 		sin6->sin6_len = sizeof(*sin6);
72958360b13Sdlg 		sin6->sin6_family = AF_INET6;
73058360b13Sdlg 		sin6->sin6_port = *port;
73158360b13Sdlg 		sin6->sin6_addr = (struct in6_addr) { .s6_addr = { 0 } };
73258360b13Sdlg 		mhostnam.m_len = sin6->sin6_len;
73358360b13Sdlg #endif
73458360b13Sdlg 	} else {
73558360b13Sdlg 		return EAFNOSUPPORT;
73658360b13Sdlg 	}
73758360b13Sdlg 
73858360b13Sdlg 	if ((ret = socreate(af, so, SOCK_DGRAM, 0)) != 0)
73958360b13Sdlg 		return ret;
74058360b13Sdlg 
74184245c07Sclaudio 	solock(*so);
74258360b13Sdlg 	sotoinpcb(*so)->inp_upcall = wg_input;
74358360b13Sdlg 	sotoinpcb(*so)->inp_upcall_arg = upcall_arg;
744ee078639Smvs 	sounlock(*so);
74558360b13Sdlg 
74658360b13Sdlg 	if ((ret = sosetopt(*so, SOL_SOCKET, SO_RTABLE, &mrtable)) == 0) {
747ee078639Smvs 		solock(*so);
74858360b13Sdlg 		if ((ret = sobind(*so, &mhostnam, curproc)) == 0) {
74958360b13Sdlg 			*port = sotoinpcb(*so)->inp_lport;
75058360b13Sdlg 			*rtable = sotoinpcb(*so)->inp_rtableid;
75158360b13Sdlg 		}
75284245c07Sclaudio 		sounlock(*so);
753ee078639Smvs 	}
75458360b13Sdlg 
75558360b13Sdlg 	if (ret != 0)
75658360b13Sdlg 		wg_socket_close(so);
75758360b13Sdlg 
75858360b13Sdlg 	return ret;
75958360b13Sdlg }
76058360b13Sdlg 
76158360b13Sdlg void
76258360b13Sdlg wg_socket_close(struct socket **so)
76358360b13Sdlg {
76458360b13Sdlg 	if (*so != NULL && soclose(*so, 0) != 0)
76558360b13Sdlg 		panic("Unable to close wg socket");
76658360b13Sdlg 	*so = NULL;
76758360b13Sdlg }
76858360b13Sdlg 
76958360b13Sdlg int
77058360b13Sdlg wg_bind(struct wg_softc *sc, in_port_t *portp, int *rtablep)
77158360b13Sdlg {
77258360b13Sdlg 	int		 ret = 0, rtable = *rtablep;
77358360b13Sdlg 	in_port_t	 port = *portp;
77458360b13Sdlg 	struct socket	*so4;
77558360b13Sdlg #ifdef INET6
77658360b13Sdlg 	struct socket	*so6;
77758360b13Sdlg 	int		 retries = 0;
77858360b13Sdlg retry:
77958360b13Sdlg #endif
78058360b13Sdlg 	if ((ret = wg_socket_open(&so4, AF_INET, &port, &rtable, sc)) != 0)
78158360b13Sdlg 		return ret;
78258360b13Sdlg 
78358360b13Sdlg #ifdef INET6
78458360b13Sdlg 	if ((ret = wg_socket_open(&so6, AF_INET6, &port, &rtable, sc)) != 0) {
78558360b13Sdlg 		if (ret == EADDRINUSE && *portp == 0 && retries++ < 100)
78658360b13Sdlg 			goto retry;
78758360b13Sdlg 		wg_socket_close(&so4);
78858360b13Sdlg 		return ret;
78958360b13Sdlg 	}
79058360b13Sdlg #endif
79158360b13Sdlg 
79258360b13Sdlg 	rw_enter_write(&sc->sc_so_lock);
79358360b13Sdlg 	wg_socket_close(&sc->sc_so4);
79458360b13Sdlg 	sc->sc_so4 = so4;
79558360b13Sdlg #ifdef INET6
79658360b13Sdlg 	wg_socket_close(&sc->sc_so6);
79758360b13Sdlg 	sc->sc_so6 = so6;
79858360b13Sdlg #endif
79958360b13Sdlg 	rw_exit_write(&sc->sc_so_lock);
80058360b13Sdlg 
80158360b13Sdlg 	*portp = port;
80258360b13Sdlg 	*rtablep = rtable;
80358360b13Sdlg 	return 0;
80458360b13Sdlg }
80558360b13Sdlg 
80658360b13Sdlg void
80758360b13Sdlg wg_unbind(struct wg_softc *sc)
80858360b13Sdlg {
80958360b13Sdlg 	rw_enter_write(&sc->sc_so_lock);
81058360b13Sdlg 	wg_socket_close(&sc->sc_so4);
81158360b13Sdlg #ifdef INET6
81258360b13Sdlg 	wg_socket_close(&sc->sc_so6);
81358360b13Sdlg #endif
81458360b13Sdlg 	rw_exit_write(&sc->sc_so_lock);
81558360b13Sdlg }
81658360b13Sdlg 
81758360b13Sdlg int
81858360b13Sdlg wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m)
81958360b13Sdlg {
82058360b13Sdlg 	struct mbuf	 peernam, *control = NULL;
82158360b13Sdlg 	int		 ret;
82258360b13Sdlg 
82358360b13Sdlg 	/* Get local control address before locking */
82458360b13Sdlg 	if (e->e_remote.r_sa.sa_family == AF_INET) {
82558360b13Sdlg 		if (e->e_local.l_in.s_addr != INADDR_ANY)
82658360b13Sdlg 			control = sbcreatecontrol(&e->e_local.l_in,
82758360b13Sdlg 			    sizeof(struct in_addr), IP_SENDSRCADDR,
82858360b13Sdlg 			    IPPROTO_IP);
82958360b13Sdlg #ifdef INET6
83058360b13Sdlg 	} else if (e->e_remote.r_sa.sa_family == AF_INET6) {
83158360b13Sdlg 		if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6))
83258360b13Sdlg 			control = sbcreatecontrol(&e->e_local.l_pktinfo6,
83358360b13Sdlg 			    sizeof(struct in6_pktinfo), IPV6_PKTINFO,
83458360b13Sdlg 			    IPPROTO_IPV6);
83558360b13Sdlg #endif
83658360b13Sdlg 	} else {
8372f84ac13Ssthen 		m_freem(m);
83858360b13Sdlg 		return EAFNOSUPPORT;
83958360b13Sdlg 	}
84058360b13Sdlg 
84158360b13Sdlg 	/* Get remote address */
84258360b13Sdlg 	peernam.m_type = MT_SONAME;
84358360b13Sdlg 	peernam.m_next = NULL;
84458360b13Sdlg 	peernam.m_nextpkt = NULL;
84558360b13Sdlg 	peernam.m_data = (void *)&e->e_remote.r_sa;
84658360b13Sdlg 	peernam.m_len = e->e_remote.r_sa.sa_len;
84758360b13Sdlg 	peernam.m_flags = 0;
84858360b13Sdlg 
84958360b13Sdlg 	rw_enter_read(&sc->sc_so_lock);
85058360b13Sdlg 	if (e->e_remote.r_sa.sa_family == AF_INET && sc->sc_so4 != NULL)
85158360b13Sdlg 		ret = sosend(sc->sc_so4, &peernam, NULL, m, control, 0);
85258360b13Sdlg #ifdef INET6
85358360b13Sdlg 	else if (e->e_remote.r_sa.sa_family == AF_INET6 && sc->sc_so6 != NULL)
85458360b13Sdlg 		ret = sosend(sc->sc_so6, &peernam, NULL, m, control, 0);
85558360b13Sdlg #endif
85658360b13Sdlg 	else {
85758360b13Sdlg 		ret = ENOTCONN;
85858360b13Sdlg 		m_freem(control);
85958360b13Sdlg 		m_freem(m);
86058360b13Sdlg 	}
86158360b13Sdlg 	rw_exit_read(&sc->sc_so_lock);
86258360b13Sdlg 
86358360b13Sdlg 	return ret;
86458360b13Sdlg }
86558360b13Sdlg 
86658360b13Sdlg void
86758360b13Sdlg wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, uint8_t *buf,
86858360b13Sdlg     size_t len)
86958360b13Sdlg {
87058360b13Sdlg 	struct mbuf	*m;
87158360b13Sdlg 	int		 ret = 0;
872c0619985Sclaudio 	size_t		 mlen = len + max_hdr;
87358360b13Sdlg 
87458360b13Sdlg retry:
87558360b13Sdlg 	m = m_gethdr(M_WAIT, MT_DATA);
876c0619985Sclaudio 	if (mlen > MHLEN)
877c0619985Sclaudio 		MCLGETL(m, M_WAIT, mlen);
878c0619985Sclaudio 	m_align(m, len);
879c0619985Sclaudio 	m->m_pkthdr.len = m->m_len = len;
880c0619985Sclaudio 	memcpy(mtod(m, void *), buf, len);
88158360b13Sdlg 
88258360b13Sdlg 	/* As we're sending a handshake packet here, we want high priority */
88358360b13Sdlg 	m->m_pkthdr.pf.prio = IFQ_MAXPRIO;
88458360b13Sdlg 
88558360b13Sdlg 	if (ret == 0) {
88658360b13Sdlg 		ret = wg_send(sc, e, m);
88758360b13Sdlg 		/* Retry if we couldn't bind to e->e_local */
88858360b13Sdlg 		if (ret == EADDRNOTAVAIL) {
88958360b13Sdlg 			bzero(&e->e_local, sizeof(e->e_local));
89058360b13Sdlg 			goto retry;
89158360b13Sdlg 		}
89258360b13Sdlg 	} else {
89358360b13Sdlg 		ret = wg_send(sc, e, m);
89458360b13Sdlg 		if (ret != 0)
895*f08653c5Smvs 			WGPRINTF(LOG_DEBUG, sc, NULL,
896*f08653c5Smvs 			    "Unable to send packet\n");
89758360b13Sdlg 	}
89858360b13Sdlg }
89958360b13Sdlg 
90058360b13Sdlg struct wg_tag *
90158360b13Sdlg wg_tag_get(struct mbuf *m)
90258360b13Sdlg {
90358360b13Sdlg 	struct m_tag	*mtag;
90458360b13Sdlg 
90558360b13Sdlg 	if ((mtag = m_tag_find(m, PACKET_TAG_WIREGUARD, NULL)) == NULL) {
90658360b13Sdlg 		mtag = m_tag_get(PACKET_TAG_WIREGUARD, sizeof(struct wg_tag),
90758360b13Sdlg 		    M_NOWAIT);
90858360b13Sdlg 		if (mtag == NULL)
90958360b13Sdlg 			return (NULL);
91058360b13Sdlg 		bzero(mtag + 1, sizeof(struct wg_tag));
91158360b13Sdlg 		m_tag_prepend(m, mtag);
91258360b13Sdlg 	}
91358360b13Sdlg 	return ((struct wg_tag *)(mtag + 1));
91458360b13Sdlg }
91558360b13Sdlg 
91658360b13Sdlg /*
91758360b13Sdlg  * The following section handles the timeout callbacks for a WireGuard session.
918678831beSjsg  * These functions provide an "event based" model for controlling wg(8) session
91958360b13Sdlg  * timers. All function calls occur after the specified event below.
92058360b13Sdlg  *
92158360b13Sdlg  * wg_timers_event_data_sent:
92258360b13Sdlg  *	tx: data
92358360b13Sdlg  * wg_timers_event_data_received:
92458360b13Sdlg  *	rx: data
92558360b13Sdlg  * wg_timers_event_any_authenticated_packet_sent:
92658360b13Sdlg  *	tx: keepalive, data, handshake
92758360b13Sdlg  * wg_timers_event_any_authenticated_packet_received:
92858360b13Sdlg  *	rx: keepalive, data, handshake
92958360b13Sdlg  * wg_timers_event_any_authenticated_packet_traversal:
93058360b13Sdlg  *	tx, rx: keepalive, data, handshake
93158360b13Sdlg  * wg_timers_event_handshake_initiated:
93258360b13Sdlg  *	tx: initiation
93358360b13Sdlg  * wg_timers_event_handshake_responded:
93458360b13Sdlg  *	tx: response
93558360b13Sdlg  * wg_timers_event_handshake_complete:
93658360b13Sdlg  *	rx: response, confirmation data
93758360b13Sdlg  * wg_timers_event_session_derived:
93858360b13Sdlg  *	tx: response, rx: response
93958360b13Sdlg  * wg_timers_event_want_initiation:
94058360b13Sdlg  *	tx: data failed, old keys expiring
94158360b13Sdlg  * wg_timers_event_reset_handshake_last_sent:
94258360b13Sdlg  * 	anytime we may immediately want a new handshake
94358360b13Sdlg  */
94458360b13Sdlg void
94558360b13Sdlg wg_timers_init(struct wg_timers *t)
94658360b13Sdlg {
94758360b13Sdlg 	bzero(t, sizeof(*t));
948efbb2e09Smvs 	mtx_init_flags(&t->t_mtx, IPL_NET, "wg_timers", 0);
94958360b13Sdlg 	mtx_init(&t->t_handshake_mtx, IPL_NET);
95058360b13Sdlg 
95158360b13Sdlg 	timeout_set(&t->t_new_handshake, wg_timers_run_new_handshake, t);
95258360b13Sdlg 	timeout_set(&t->t_send_keepalive, wg_timers_run_send_keepalive, t);
95358360b13Sdlg 	timeout_set(&t->t_retry_handshake, wg_timers_run_retry_handshake, t);
95458360b13Sdlg 	timeout_set(&t->t_persistent_keepalive,
95558360b13Sdlg 	    wg_timers_run_persistent_keepalive, t);
95658360b13Sdlg 	timeout_set(&t->t_zero_key_material,
95758360b13Sdlg 	    wg_timers_run_zero_key_material, t);
95858360b13Sdlg }
95958360b13Sdlg 
96058360b13Sdlg void
96158360b13Sdlg wg_timers_enable(struct wg_timers *t)
96258360b13Sdlg {
963efbb2e09Smvs 	mtx_enter(&t->t_mtx);
96458360b13Sdlg 	t->t_disabled = 0;
965efbb2e09Smvs 	mtx_leave(&t->t_mtx);
96658360b13Sdlg 	wg_timers_run_persistent_keepalive(t);
96758360b13Sdlg }
96858360b13Sdlg 
96958360b13Sdlg void
97058360b13Sdlg wg_timers_disable(struct wg_timers *t)
97158360b13Sdlg {
972efbb2e09Smvs 	mtx_enter(&t->t_mtx);
97358360b13Sdlg 	t->t_disabled = 1;
97458360b13Sdlg 	t->t_need_another_keepalive = 0;
975efbb2e09Smvs 	mtx_leave(&t->t_mtx);
97658360b13Sdlg 
97758360b13Sdlg 	timeout_del_barrier(&t->t_new_handshake);
97858360b13Sdlg 	timeout_del_barrier(&t->t_send_keepalive);
97958360b13Sdlg 	timeout_del_barrier(&t->t_retry_handshake);
98058360b13Sdlg 	timeout_del_barrier(&t->t_persistent_keepalive);
98158360b13Sdlg 	timeout_del_barrier(&t->t_zero_key_material);
98258360b13Sdlg }
98358360b13Sdlg 
98458360b13Sdlg void
98558360b13Sdlg wg_timers_set_persistent_keepalive(struct wg_timers *t, uint16_t interval)
98658360b13Sdlg {
987efbb2e09Smvs 	mtx_enter(&t->t_mtx);
98858360b13Sdlg 	if (!t->t_disabled) {
98958360b13Sdlg 		t->t_persistent_keepalive_interval = interval;
99058360b13Sdlg 		wg_timers_run_persistent_keepalive(t);
99158360b13Sdlg 	}
992efbb2e09Smvs 	mtx_leave(&t->t_mtx);
99358360b13Sdlg }
99458360b13Sdlg 
99558360b13Sdlg int
99658360b13Sdlg wg_timers_get_persistent_keepalive(struct wg_timers *t, uint16_t *interval)
99758360b13Sdlg {
99858360b13Sdlg 	*interval = t->t_persistent_keepalive_interval;
99958360b13Sdlg 	return *interval > 0 ? 0 : ENOENT;
100058360b13Sdlg }
100158360b13Sdlg 
100258360b13Sdlg void
100358360b13Sdlg wg_timers_get_last_handshake(struct wg_timers *t, struct timespec *time)
100458360b13Sdlg {
100558360b13Sdlg 	mtx_enter(&t->t_handshake_mtx);
100658360b13Sdlg 	*time = t->t_handshake_complete;
100758360b13Sdlg 	mtx_leave(&t->t_handshake_mtx);
100858360b13Sdlg }
100958360b13Sdlg 
101058360b13Sdlg int
101158360b13Sdlg wg_timers_expired_handshake_last_sent(struct wg_timers *t)
101258360b13Sdlg {
101358360b13Sdlg 	struct timespec uptime;
101458360b13Sdlg 	struct timespec expire = { .tv_sec = REKEY_TIMEOUT, .tv_nsec = 0 };
101558360b13Sdlg 
101658360b13Sdlg 	getnanouptime(&uptime);
101758360b13Sdlg 	timespecadd(&t->t_handshake_last_sent, &expire, &expire);
101858360b13Sdlg 	return timespeccmp(&uptime, &expire, >) ? ETIMEDOUT : 0;
101958360b13Sdlg }
102058360b13Sdlg 
102158360b13Sdlg int
102258360b13Sdlg wg_timers_check_handshake_last_sent(struct wg_timers *t)
102358360b13Sdlg {
102458360b13Sdlg 	int ret;
102558360b13Sdlg 	mtx_enter(&t->t_handshake_mtx);
102658360b13Sdlg 	if ((ret = wg_timers_expired_handshake_last_sent(t)) == ETIMEDOUT)
102758360b13Sdlg 		getnanouptime(&t->t_handshake_last_sent);
102858360b13Sdlg 	mtx_leave(&t->t_handshake_mtx);
102958360b13Sdlg 	return ret;
103058360b13Sdlg }
103158360b13Sdlg 
103258360b13Sdlg void
103358360b13Sdlg wg_timers_event_data_sent(struct wg_timers *t)
103458360b13Sdlg {
103558360b13Sdlg 	int	msecs = NEW_HANDSHAKE_TIMEOUT * 1000;
103658360b13Sdlg 	msecs += arc4random_uniform(REKEY_TIMEOUT_JITTER);
103758360b13Sdlg 
1038efbb2e09Smvs 	mtx_enter(&t->t_mtx);
103958360b13Sdlg 	if (!t->t_disabled && !timeout_pending(&t->t_new_handshake))
104058360b13Sdlg 		timeout_add_msec(&t->t_new_handshake, msecs);
1041efbb2e09Smvs 	mtx_leave(&t->t_mtx);
104258360b13Sdlg }
104358360b13Sdlg 
104458360b13Sdlg void
104558360b13Sdlg wg_timers_event_data_received(struct wg_timers *t)
104658360b13Sdlg {
1047efbb2e09Smvs 	mtx_enter(&t->t_mtx);
104858360b13Sdlg 	if (!t->t_disabled) {
104958360b13Sdlg 		if (!timeout_pending(&t->t_send_keepalive))
105058360b13Sdlg 			timeout_add_sec(&t->t_send_keepalive,
105158360b13Sdlg 			    KEEPALIVE_TIMEOUT);
105258360b13Sdlg 		else
105358360b13Sdlg 			t->t_need_another_keepalive = 1;
105458360b13Sdlg 	}
1055efbb2e09Smvs 	mtx_leave(&t->t_mtx);
105658360b13Sdlg }
105758360b13Sdlg 
105858360b13Sdlg void
105958360b13Sdlg wg_timers_event_any_authenticated_packet_sent(struct wg_timers *t)
106058360b13Sdlg {
106158360b13Sdlg 	timeout_del(&t->t_send_keepalive);
106258360b13Sdlg }
106358360b13Sdlg 
106458360b13Sdlg void
106558360b13Sdlg wg_timers_event_any_authenticated_packet_received(struct wg_timers *t)
106658360b13Sdlg {
106758360b13Sdlg 	timeout_del(&t->t_new_handshake);
106858360b13Sdlg }
106958360b13Sdlg 
107058360b13Sdlg void
107158360b13Sdlg wg_timers_event_any_authenticated_packet_traversal(struct wg_timers *t)
107258360b13Sdlg {
1073efbb2e09Smvs 	mtx_enter(&t->t_mtx);
107458360b13Sdlg 	if (!t->t_disabled && t->t_persistent_keepalive_interval > 0)
107558360b13Sdlg 		timeout_add_sec(&t->t_persistent_keepalive,
107658360b13Sdlg 		    t->t_persistent_keepalive_interval);
1077efbb2e09Smvs 	mtx_leave(&t->t_mtx);
107858360b13Sdlg }
107958360b13Sdlg 
108058360b13Sdlg void
108158360b13Sdlg wg_timers_event_handshake_initiated(struct wg_timers *t)
108258360b13Sdlg {
108358360b13Sdlg 	int	msecs = REKEY_TIMEOUT * 1000;
108458360b13Sdlg 	msecs += arc4random_uniform(REKEY_TIMEOUT_JITTER);
108558360b13Sdlg 
1086efbb2e09Smvs 	mtx_enter(&t->t_mtx);
108758360b13Sdlg 	if (!t->t_disabled)
108858360b13Sdlg 		timeout_add_msec(&t->t_retry_handshake, msecs);
1089efbb2e09Smvs 	mtx_leave(&t->t_mtx);
109058360b13Sdlg }
109158360b13Sdlg 
109258360b13Sdlg void
109358360b13Sdlg wg_timers_event_handshake_responded(struct wg_timers *t)
109458360b13Sdlg {
109558360b13Sdlg 	mtx_enter(&t->t_handshake_mtx);
109658360b13Sdlg 	getnanouptime(&t->t_handshake_last_sent);
109758360b13Sdlg 	mtx_leave(&t->t_handshake_mtx);
109858360b13Sdlg }
109958360b13Sdlg 
110058360b13Sdlg void
110158360b13Sdlg wg_timers_event_handshake_complete(struct wg_timers *t)
110258360b13Sdlg {
1103efbb2e09Smvs 	mtx_enter(&t->t_mtx);
110458360b13Sdlg 	if (!t->t_disabled) {
110558360b13Sdlg 		mtx_enter(&t->t_handshake_mtx);
110658360b13Sdlg 		timeout_del(&t->t_retry_handshake);
110758360b13Sdlg 		t->t_handshake_retries = 0;
110858360b13Sdlg 		getnanotime(&t->t_handshake_complete);
110958360b13Sdlg 		mtx_leave(&t->t_handshake_mtx);
111058360b13Sdlg 		wg_timers_run_send_keepalive(t);
111158360b13Sdlg 	}
1112efbb2e09Smvs 	mtx_leave(&t->t_mtx);
111358360b13Sdlg }
111458360b13Sdlg 
111558360b13Sdlg void
111658360b13Sdlg wg_timers_event_session_derived(struct wg_timers *t)
111758360b13Sdlg {
1118efbb2e09Smvs 	mtx_enter(&t->t_mtx);
111958360b13Sdlg 	if (!t->t_disabled)
112058360b13Sdlg 		timeout_add_sec(&t->t_zero_key_material, REJECT_AFTER_TIME * 3);
1121efbb2e09Smvs 	mtx_leave(&t->t_mtx);
112258360b13Sdlg }
112358360b13Sdlg 
112458360b13Sdlg void
112558360b13Sdlg wg_timers_event_want_initiation(struct wg_timers *t)
112658360b13Sdlg {
1127efbb2e09Smvs 	mtx_enter(&t->t_mtx);
112858360b13Sdlg 	if (!t->t_disabled)
112958360b13Sdlg 		wg_timers_run_send_initiation(t, 0);
1130efbb2e09Smvs 	mtx_leave(&t->t_mtx);
113158360b13Sdlg }
113258360b13Sdlg 
113358360b13Sdlg void
113458360b13Sdlg wg_timers_event_reset_handshake_last_sent(struct wg_timers *t)
113558360b13Sdlg {
113658360b13Sdlg 	mtx_enter(&t->t_handshake_mtx);
113758360b13Sdlg 	t->t_handshake_last_sent.tv_sec -= (REKEY_TIMEOUT + 1);
113858360b13Sdlg 	mtx_leave(&t->t_handshake_mtx);
113958360b13Sdlg }
114058360b13Sdlg 
114158360b13Sdlg void
114258360b13Sdlg wg_timers_run_send_initiation(void *_t, int is_retry)
114358360b13Sdlg {
114458360b13Sdlg 	struct wg_timers *t = _t;
114558360b13Sdlg 	struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers);
114658360b13Sdlg 	if (!is_retry)
114758360b13Sdlg 		t->t_handshake_retries = 0;
114858360b13Sdlg 	if (wg_timers_expired_handshake_last_sent(t) == ETIMEDOUT)
114958360b13Sdlg 		task_add(wg_handshake_taskq, &peer->p_send_initiation);
115058360b13Sdlg }
115158360b13Sdlg 
115258360b13Sdlg void
115358360b13Sdlg wg_timers_run_retry_handshake(void *_t)
115458360b13Sdlg {
115558360b13Sdlg 	struct wg_timers *t = _t;
115658360b13Sdlg 	struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers);
1157*f08653c5Smvs 	char		  ipaddr[INET6_ADDRSTRLEN];
115858360b13Sdlg 
115958360b13Sdlg 	mtx_enter(&t->t_handshake_mtx);
116058360b13Sdlg 	if (t->t_handshake_retries <= MAX_TIMER_HANDSHAKES) {
116158360b13Sdlg 		t->t_handshake_retries++;
116258360b13Sdlg 		mtx_leave(&t->t_handshake_mtx);
116358360b13Sdlg 
1164*f08653c5Smvs 		WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx,
1165*f08653c5Smvs 		    "Handshake for peer %llu (%s) did not complete after %d "
1166*f08653c5Smvs 		    "seconds, retrying (try %d)\n", peer->p_id,
1167*f08653c5Smvs 		    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1168*f08653c5Smvs 		        sizeof(ipaddr)),
116958360b13Sdlg 		    REKEY_TIMEOUT, t->t_handshake_retries + 1);
117058360b13Sdlg 		wg_peer_clear_src(peer);
117158360b13Sdlg 		wg_timers_run_send_initiation(t, 1);
117258360b13Sdlg 	} else {
117358360b13Sdlg 		mtx_leave(&t->t_handshake_mtx);
117458360b13Sdlg 
1175*f08653c5Smvs 		WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx,
1176*f08653c5Smvs 		    "Handshake for peer %llu (%s) did not complete after %d "
1177*f08653c5Smvs 		    "retries, giving up\n", peer->p_id,
1178*f08653c5Smvs 		    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1179*f08653c5Smvs 		        sizeof(ipaddr)), MAX_TIMER_HANDSHAKES + 2);
118058360b13Sdlg 
118158360b13Sdlg 		timeout_del(&t->t_send_keepalive);
118258360b13Sdlg 		mq_purge(&peer->p_stage_queue);
118358360b13Sdlg 		if (!timeout_pending(&t->t_zero_key_material))
118458360b13Sdlg 			timeout_add_sec(&t->t_zero_key_material,
118558360b13Sdlg 			    REJECT_AFTER_TIME * 3);
118658360b13Sdlg 	}
118758360b13Sdlg }
118858360b13Sdlg 
118958360b13Sdlg void
119058360b13Sdlg wg_timers_run_send_keepalive(void *_t)
119158360b13Sdlg {
119258360b13Sdlg 	struct wg_timers *t = _t;
119358360b13Sdlg 	struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers);
119458360b13Sdlg 
119558360b13Sdlg 	task_add(wg_crypt_taskq, &peer->p_send_keepalive);
119658360b13Sdlg 	if (t->t_need_another_keepalive) {
119758360b13Sdlg 		t->t_need_another_keepalive = 0;
119858360b13Sdlg 		timeout_add_sec(&t->t_send_keepalive, KEEPALIVE_TIMEOUT);
119958360b13Sdlg 	}
120058360b13Sdlg }
120158360b13Sdlg 
120258360b13Sdlg void
120358360b13Sdlg wg_timers_run_new_handshake(void *_t)
120458360b13Sdlg {
120558360b13Sdlg 	struct wg_timers *t = _t;
120658360b13Sdlg 	struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers);
1207*f08653c5Smvs 	char		  ipaddr[INET6_ADDRSTRLEN];
120858360b13Sdlg 
1209*f08653c5Smvs 	WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx,
1210*f08653c5Smvs 	    "Retrying handshake with peer %llu (%s) because we "
1211*f08653c5Smvs 	    "stopped hearing back after %d seconds\n", peer->p_id,
1212*f08653c5Smvs 	    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1213*f08653c5Smvs 	        sizeof(ipaddr)), NEW_HANDSHAKE_TIMEOUT);
121458360b13Sdlg 	wg_peer_clear_src(peer);
121558360b13Sdlg 
121658360b13Sdlg 	wg_timers_run_send_initiation(t, 0);
121758360b13Sdlg }
121858360b13Sdlg 
121958360b13Sdlg void
122058360b13Sdlg wg_timers_run_zero_key_material(void *_t)
122158360b13Sdlg {
122258360b13Sdlg 	struct wg_timers *t = _t;
122358360b13Sdlg 	struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers);
1224*f08653c5Smvs 	char		  ipaddr[INET6_ADDRSTRLEN];
122558360b13Sdlg 
1226*f08653c5Smvs 	WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, "Zeroing out "
1227*f08653c5Smvs 	    "keys for peer %llu (%s)\n", peer->p_id,
1228*f08653c5Smvs 	    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1229*f08653c5Smvs 	        sizeof(ipaddr)));
123058360b13Sdlg 	task_add(wg_handshake_taskq, &peer->p_clear_secrets);
123158360b13Sdlg }
123258360b13Sdlg 
123358360b13Sdlg void
123458360b13Sdlg wg_timers_run_persistent_keepalive(void *_t)
123558360b13Sdlg {
123658360b13Sdlg 	struct wg_timers *t = _t;
123758360b13Sdlg 	struct wg_peer	 *peer = CONTAINER_OF(t, struct wg_peer, p_timers);
123858360b13Sdlg 	if (t->t_persistent_keepalive_interval != 0)
123958360b13Sdlg 		task_add(wg_crypt_taskq, &peer->p_send_keepalive);
124058360b13Sdlg }
124158360b13Sdlg 
124258360b13Sdlg /* The following functions handle handshakes */
124358360b13Sdlg void
124458360b13Sdlg wg_peer_send_buf(struct wg_peer *peer, uint8_t *buf, size_t len)
124558360b13Sdlg {
124658360b13Sdlg 	struct wg_endpoint	 endpoint;
124758360b13Sdlg 
124858360b13Sdlg 	wg_peer_counters_add(peer, len, 0);
124958360b13Sdlg 	wg_timers_event_any_authenticated_packet_traversal(&peer->p_timers);
125058360b13Sdlg 	wg_timers_event_any_authenticated_packet_sent(&peer->p_timers);
125158360b13Sdlg 	wg_peer_get_endpoint(peer, &endpoint);
125258360b13Sdlg 	wg_send_buf(peer->p_sc, &endpoint, buf, len);
125358360b13Sdlg }
125458360b13Sdlg 
125558360b13Sdlg void
125658360b13Sdlg wg_send_initiation(void *_peer)
125758360b13Sdlg {
125858360b13Sdlg 	struct wg_peer			*peer = _peer;
125958360b13Sdlg 	struct wg_pkt_initiation	 pkt;
1260*f08653c5Smvs 	char				 ipaddr[INET6_ADDRSTRLEN];
126158360b13Sdlg 
126258360b13Sdlg 	if (wg_timers_check_handshake_last_sent(&peer->p_timers) != ETIMEDOUT)
126358360b13Sdlg 		return;
126458360b13Sdlg 
1265*f08653c5Smvs 	WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, "Sending "
1266*f08653c5Smvs 	    "handshake initiation to peer %llu (%s)\n", peer->p_id,
1267*f08653c5Smvs 	    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1268*f08653c5Smvs 	        sizeof(ipaddr)));
126958360b13Sdlg 
127058360b13Sdlg 	if (noise_create_initiation(&peer->p_remote, &pkt.s_idx, pkt.ue, pkt.es,
127158360b13Sdlg 				    pkt.ets) != 0)
127258360b13Sdlg 		return;
127358360b13Sdlg 	pkt.t = WG_PKT_INITIATION;
127458360b13Sdlg 	cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
127558360b13Sdlg 	    sizeof(pkt)-sizeof(pkt.m));
127658360b13Sdlg 	wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt));
127758360b13Sdlg 	wg_timers_event_handshake_initiated(&peer->p_timers);
127858360b13Sdlg }
127958360b13Sdlg 
128058360b13Sdlg void
128158360b13Sdlg wg_send_response(struct wg_peer *peer)
128258360b13Sdlg {
128358360b13Sdlg 	struct wg_pkt_response	 pkt;
1284*f08653c5Smvs 	char			 ipaddr[INET6_ADDRSTRLEN];
128558360b13Sdlg 
1286*f08653c5Smvs 	WGPRINTF(LOG_INFO, peer->p_sc, &peer->p_endpoint_mtx, "Sending "
1287*f08653c5Smvs 	    "handshake response to peer %llu (%s)\n", peer->p_id,
1288*f08653c5Smvs 	    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1289*f08653c5Smvs 	        sizeof(ipaddr)));
129058360b13Sdlg 
129158360b13Sdlg 	if (noise_create_response(&peer->p_remote, &pkt.s_idx, &pkt.r_idx,
129258360b13Sdlg 				  pkt.ue, pkt.en) != 0)
129358360b13Sdlg 		return;
129458360b13Sdlg 	if (noise_remote_begin_session(&peer->p_remote) != 0)
129558360b13Sdlg 		return;
129658360b13Sdlg 	wg_timers_event_session_derived(&peer->p_timers);
129758360b13Sdlg 	pkt.t = WG_PKT_RESPONSE;
129858360b13Sdlg 	cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
129958360b13Sdlg 	    sizeof(pkt)-sizeof(pkt.m));
130058360b13Sdlg 	wg_timers_event_handshake_responded(&peer->p_timers);
130158360b13Sdlg 	wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt));
130258360b13Sdlg }
130358360b13Sdlg 
130458360b13Sdlg void
130558360b13Sdlg wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx,
130658360b13Sdlg     struct wg_endpoint *e)
130758360b13Sdlg {
130858360b13Sdlg 	struct wg_pkt_cookie	pkt;
130958360b13Sdlg 
1310*f08653c5Smvs 	WGPRINTF(LOG_DEBUG, sc, NULL, "Sending cookie response for denied "
1311*f08653c5Smvs 	    "handshake message\n");
131258360b13Sdlg 
131358360b13Sdlg 	pkt.t = WG_PKT_COOKIE;
131458360b13Sdlg 	pkt.r_idx = idx;
131558360b13Sdlg 
131658360b13Sdlg 	cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce,
131758360b13Sdlg 	    pkt.ec, &e->e_remote.r_sa);
131858360b13Sdlg 
131958360b13Sdlg 	wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt));
132058360b13Sdlg }
132158360b13Sdlg 
132258360b13Sdlg void
132358360b13Sdlg wg_send_keepalive(void *_peer)
132458360b13Sdlg {
132558360b13Sdlg 	struct wg_peer	*peer = _peer;
132658360b13Sdlg 	struct wg_softc	*sc = peer->p_sc;
132758360b13Sdlg 	struct wg_tag	*t;
132858360b13Sdlg 	struct mbuf	*m;
132958360b13Sdlg 
133058360b13Sdlg 	if (!mq_empty(&peer->p_stage_queue))
133158360b13Sdlg 		goto send;
133258360b13Sdlg 
133358360b13Sdlg 	if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
133458360b13Sdlg 		return;
133558360b13Sdlg 
133658360b13Sdlg 	if ((t = wg_tag_get(m)) == NULL) {
133758360b13Sdlg 		m_freem(m);
133858360b13Sdlg 		return;
133958360b13Sdlg 	}
134058360b13Sdlg 
134158360b13Sdlg 	t->t_peer = peer;
134258360b13Sdlg 	t->t_mbuf = NULL;
134358360b13Sdlg 	t->t_done = 0;
134458360b13Sdlg 	t->t_mtu = 0; /* MTU == 0 OK for keepalive */
134558360b13Sdlg 
134658360b13Sdlg 	mq_push(&peer->p_stage_queue, m);
134758360b13Sdlg send:
134858360b13Sdlg 	if (noise_remote_ready(&peer->p_remote) == 0) {
134958360b13Sdlg 		wg_queue_out(sc, peer);
135058360b13Sdlg 		task_add(wg_crypt_taskq, &sc->sc_encap);
135158360b13Sdlg 	} else {
135258360b13Sdlg 		wg_timers_event_want_initiation(&peer->p_timers);
135358360b13Sdlg 	}
135458360b13Sdlg }
135558360b13Sdlg 
135658360b13Sdlg void
135758360b13Sdlg wg_peer_clear_secrets(void *_peer)
135858360b13Sdlg {
135958360b13Sdlg 	struct wg_peer *peer = _peer;
136058360b13Sdlg 	noise_remote_clear(&peer->p_remote);
136158360b13Sdlg }
136258360b13Sdlg 
136358360b13Sdlg void
136458360b13Sdlg wg_handshake(struct wg_softc *sc, struct mbuf *m)
136558360b13Sdlg {
136658360b13Sdlg 	struct wg_tag			*t;
136758360b13Sdlg 	struct wg_pkt_initiation	*init;
136858360b13Sdlg 	struct wg_pkt_response		*resp;
136958360b13Sdlg 	struct wg_pkt_cookie		*cook;
137058360b13Sdlg 	struct wg_peer			*peer;
137158360b13Sdlg 	struct noise_remote		*remote;
137258360b13Sdlg 	int				 res, underload = 0;
137358360b13Sdlg 	static struct timeval		 wg_last_underload; /* microuptime */
1374*f08653c5Smvs 	char				 ipaddr[INET6_ADDRSTRLEN];
137558360b13Sdlg 
137658360b13Sdlg 	if (mq_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES/8) {
137758360b13Sdlg 		getmicrouptime(&wg_last_underload);
137858360b13Sdlg 		underload = 1;
137958360b13Sdlg 	} else if (wg_last_underload.tv_sec != 0) {
138058360b13Sdlg 		if (!ratecheck(&wg_last_underload, &underload_interval))
138158360b13Sdlg 			underload = 1;
138258360b13Sdlg 		else
138358360b13Sdlg 			bzero(&wg_last_underload, sizeof(wg_last_underload));
138458360b13Sdlg 	}
138558360b13Sdlg 
138658360b13Sdlg 	t = wg_tag_get(m);
138758360b13Sdlg 
138858360b13Sdlg 	switch (*mtod(m, uint32_t *)) {
138958360b13Sdlg 	case WG_PKT_INITIATION:
139058360b13Sdlg 		init = mtod(m, struct wg_pkt_initiation *);
139158360b13Sdlg 
139258360b13Sdlg 		res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m,
139358360b13Sdlg 				init, sizeof(*init) - sizeof(init->m),
139458360b13Sdlg 				underload, &t->t_endpoint.e_remote.r_sa);
139558360b13Sdlg 
139658360b13Sdlg 		if (res == EINVAL) {
1397*f08653c5Smvs 			WGPRINTF(LOG_INFO, sc, NULL, "Invalid initiation "
1398*f08653c5Smvs 			    "MAC from %s\n",
1399*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1400*f08653c5Smvs 			        sizeof(ipaddr)));
140158360b13Sdlg 			goto error;
140258360b13Sdlg 		} else if (res == ECONNREFUSED) {
1403*f08653c5Smvs 			WGPRINTF(LOG_DEBUG, sc, NULL, "Handshake "
1404*f08653c5Smvs 			    "ratelimited from %s\n",
1405*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1406*f08653c5Smvs 			        sizeof(ipaddr)));
140758360b13Sdlg 			goto error;
140858360b13Sdlg 		} else if (res == EAGAIN) {
140958360b13Sdlg 			wg_send_cookie(sc, &init->m, init->s_idx,
141058360b13Sdlg 			    &t->t_endpoint);
141158360b13Sdlg 			goto error;
141258360b13Sdlg 		} else if (res != 0) {
14134123b6a7Sderaadt 			panic("unexpected response: %d", res);
141458360b13Sdlg 		}
141558360b13Sdlg 
141658360b13Sdlg 		if (noise_consume_initiation(&sc->sc_local, &remote,
141758360b13Sdlg 		    init->s_idx, init->ue, init->es, init->ets) != 0) {
1418*f08653c5Smvs 			WGPRINTF(LOG_INFO, sc, NULL, "Invalid handshake "
1419*f08653c5Smvs 			    "initiation from %s\n",
1420*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1421*f08653c5Smvs 			        sizeof(ipaddr)));
142258360b13Sdlg 			goto error;
142358360b13Sdlg 		}
142458360b13Sdlg 
142558360b13Sdlg 		peer = CONTAINER_OF(remote, struct wg_peer, p_remote);
142658360b13Sdlg 
1427*f08653c5Smvs 		WGPRINTF(LOG_INFO, sc, NULL, "Receiving handshake initiation "
1428*f08653c5Smvs 		    "from peer %llu (%s)\n", peer->p_id,
1429*f08653c5Smvs 		    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1430*f08653c5Smvs 		        sizeof(ipaddr)));
143158360b13Sdlg 
143258360b13Sdlg 		wg_peer_counters_add(peer, 0, sizeof(*init));
143358360b13Sdlg 		wg_peer_set_endpoint_from_tag(peer, t);
143458360b13Sdlg 		wg_send_response(peer);
143558360b13Sdlg 		break;
143658360b13Sdlg 	case WG_PKT_RESPONSE:
143758360b13Sdlg 		resp = mtod(m, struct wg_pkt_response *);
143858360b13Sdlg 
143958360b13Sdlg 		res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m,
144058360b13Sdlg 				resp, sizeof(*resp) - sizeof(resp->m),
144158360b13Sdlg 				underload, &t->t_endpoint.e_remote.r_sa);
144258360b13Sdlg 
144358360b13Sdlg 		if (res == EINVAL) {
1444*f08653c5Smvs 			WGPRINTF(LOG_INFO, sc, NULL, "Invalid response "
1445*f08653c5Smvs 			    "MAC from %s\n",
1446*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1447*f08653c5Smvs 			        sizeof(ipaddr)));
144858360b13Sdlg 			goto error;
144958360b13Sdlg 		} else if (res == ECONNREFUSED) {
1450*f08653c5Smvs 			WGPRINTF(LOG_DEBUG, sc, NULL, "Handshake "
1451*f08653c5Smvs 			    "ratelimited from %s\n",
1452*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1453*f08653c5Smvs 			        sizeof(ipaddr)));
145458360b13Sdlg 			goto error;
145558360b13Sdlg 		} else if (res == EAGAIN) {
145658360b13Sdlg 			wg_send_cookie(sc, &resp->m, resp->s_idx,
145758360b13Sdlg 			    &t->t_endpoint);
145858360b13Sdlg 			goto error;
145958360b13Sdlg 		} else if (res != 0) {
14604123b6a7Sderaadt 			panic("unexpected response: %d", res);
146158360b13Sdlg 		}
146258360b13Sdlg 
146358360b13Sdlg 		if ((remote = wg_index_get(sc, resp->r_idx)) == NULL) {
1464*f08653c5Smvs 			WGPRINTF(LOG_INFO, sc, NULL, "Unknown "
1465*f08653c5Smvs 			    "handshake response from %s\n",
1466*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1467*f08653c5Smvs 			        sizeof(ipaddr)));
146858360b13Sdlg 			goto error;
146958360b13Sdlg 		}
147058360b13Sdlg 
147158360b13Sdlg 		peer = CONTAINER_OF(remote, struct wg_peer, p_remote);
147258360b13Sdlg 
147358360b13Sdlg 		if (noise_consume_response(remote, resp->s_idx, resp->r_idx,
147458360b13Sdlg 					   resp->ue, resp->en) != 0) {
1475*f08653c5Smvs 			WGPRINTF(LOG_INFO, sc, NULL, "Invalid handshake "
1476*f08653c5Smvs 			    "response from %s\n",
1477*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1478*f08653c5Smvs 			        sizeof(ipaddr)));
147958360b13Sdlg 			goto error;
148058360b13Sdlg 		}
148158360b13Sdlg 
1482*f08653c5Smvs 		WGPRINTF(LOG_INFO, sc, NULL, "Receiving handshake response "
1483*f08653c5Smvs 		    "from peer %llu (%s)\n", peer->p_id,
1484*f08653c5Smvs 		    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1485*f08653c5Smvs 		        sizeof(ipaddr)));
148658360b13Sdlg 
148758360b13Sdlg 		wg_peer_counters_add(peer, 0, sizeof(*resp));
148858360b13Sdlg 		wg_peer_set_endpoint_from_tag(peer, t);
148958360b13Sdlg 		if (noise_remote_begin_session(&peer->p_remote) == 0) {
149058360b13Sdlg 			wg_timers_event_session_derived(&peer->p_timers);
149158360b13Sdlg 			wg_timers_event_handshake_complete(&peer->p_timers);
149258360b13Sdlg 		}
149358360b13Sdlg 		break;
149458360b13Sdlg 	case WG_PKT_COOKIE:
149558360b13Sdlg 		cook = mtod(m, struct wg_pkt_cookie *);
149658360b13Sdlg 
149758360b13Sdlg 		if ((remote = wg_index_get(sc, cook->r_idx)) == NULL) {
1498*f08653c5Smvs 			WGPRINTF(LOG_DEBUG, sc, NULL, "Unknown cookie "
1499*f08653c5Smvs 			    "index from %s\n",
1500*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1501*f08653c5Smvs 			        sizeof(ipaddr)));
150258360b13Sdlg 			goto error;
150358360b13Sdlg 		}
150458360b13Sdlg 
150558360b13Sdlg 		peer = CONTAINER_OF(remote, struct wg_peer, p_remote);
150658360b13Sdlg 
150758360b13Sdlg 		if (cookie_maker_consume_payload(&peer->p_cookie,
150858360b13Sdlg 		    cook->nonce, cook->ec) != 0) {
1509*f08653c5Smvs 			WGPRINTF(LOG_DEBUG, sc, NULL, "Could not decrypt "
1510*f08653c5Smvs 			    "cookie response from %s\n",
1511*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1512*f08653c5Smvs 			        sizeof(ipaddr)));
151358360b13Sdlg 			goto error;
151458360b13Sdlg 		}
151558360b13Sdlg 
1516*f08653c5Smvs 		WGPRINTF(LOG_DEBUG, sc, NULL, "Receiving cookie response "
1517*f08653c5Smvs 		    "from %s\n",
1518*f08653c5Smvs 		    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
1519*f08653c5Smvs 		        sizeof(ipaddr)));
152058360b13Sdlg 		goto error;
152158360b13Sdlg 	default:
152258360b13Sdlg 		panic("invalid packet in handshake queue");
152358360b13Sdlg 	}
152458360b13Sdlg 
152558360b13Sdlg 	wg_timers_event_any_authenticated_packet_received(&peer->p_timers);
152658360b13Sdlg 	wg_timers_event_any_authenticated_packet_traversal(&peer->p_timers);
152758360b13Sdlg error:
152858360b13Sdlg 	m_freem(m);
152958360b13Sdlg }
153058360b13Sdlg 
153158360b13Sdlg void
153258360b13Sdlg wg_handshake_worker(void *_sc)
153358360b13Sdlg {
153458360b13Sdlg 	struct mbuf *m;
153558360b13Sdlg 	struct wg_softc *sc = _sc;
153658360b13Sdlg 	while ((m = mq_dequeue(&sc->sc_handshake_queue)) != NULL)
153758360b13Sdlg 		wg_handshake(sc, m);
153858360b13Sdlg }
153958360b13Sdlg 
154058360b13Sdlg /*
154158360b13Sdlg  * The following functions handle encapsulation (encryption) and
154258360b13Sdlg  * decapsulation (decryption). The wg_{en,de}cap functions will run in the
154358360b13Sdlg  * sc_crypt_taskq, while wg_deliver_{in,out} must be serialised and will run
154458360b13Sdlg  * in nettq.
154558360b13Sdlg  *
154658360b13Sdlg  * The packets are tracked in two queues, a serial queue and a parallel queue.
154758360b13Sdlg  *  - The parallel queue is used to distribute the encryption across multiple
154858360b13Sdlg  *    threads.
154958360b13Sdlg  *  - The serial queue ensures that packets are not reordered and are
1550b076d4fbSjsg  *    delivered in sequence.
155158360b13Sdlg  * The wg_tag attached to the packet contains two flags to help the two queues
155258360b13Sdlg  * interact.
155358360b13Sdlg  *  - t_done: The parallel queue has finished with the packet, now the serial
155458360b13Sdlg  *            queue can do it's work.
155558360b13Sdlg  *  - t_mbuf: Used to store the *crypted packet. in the case of encryption,
155658360b13Sdlg  *            this is a newly allocated packet, and in the case of decryption,
155758360b13Sdlg  *            it is a pointer to the same packet, that has been decrypted and
155858360b13Sdlg  *            truncated. If t_mbuf is NULL, then *cryption failed and this
155958360b13Sdlg  *            packet should not be passed.
156058360b13Sdlg  * wg_{en,de}cap work on the parallel queue, while wg_deliver_{in,out} work
156158360b13Sdlg  * on the serial queue.
156258360b13Sdlg  */
156358360b13Sdlg void
156458360b13Sdlg wg_encap(struct wg_softc *sc, struct mbuf *m)
156558360b13Sdlg {
156658360b13Sdlg 	int res = 0;
156758360b13Sdlg 	struct wg_pkt_data	*data;
156858360b13Sdlg 	struct wg_peer		*peer;
156958360b13Sdlg 	struct wg_tag		*t;
157058360b13Sdlg 	struct mbuf		*mc;
157158360b13Sdlg 	size_t			 padding_len, plaintext_len, out_len;
157258360b13Sdlg 	uint64_t		 nonce;
1573*f08653c5Smvs 	char			 ipaddr[INET6_ADDRSTRLEN];
157458360b13Sdlg 
157558360b13Sdlg 	t = wg_tag_get(m);
157658360b13Sdlg 	peer = t->t_peer;
157758360b13Sdlg 
1578c0619985Sclaudio 	plaintext_len = WG_PKT_WITH_PADDING(m->m_pkthdr.len);
157958360b13Sdlg 	padding_len = plaintext_len - m->m_pkthdr.len;
1580c0619985Sclaudio 	out_len = sizeof(struct wg_pkt_data) + plaintext_len +
1581c0619985Sclaudio 	    NOISE_AUTHTAG_LEN;
158258360b13Sdlg 
158358360b13Sdlg 	/*
158458360b13Sdlg 	 * For the time being we allocate a new packet with sufficient size to
158558360b13Sdlg 	 * hold the encrypted data and headers. It would be difficult to
158658360b13Sdlg 	 * overcome as p_encap_queue (mbuf_list) holds a reference to the mbuf.
158758360b13Sdlg 	 * If we m_makespace or similar, we risk corrupting that list.
158858360b13Sdlg 	 * Additionally, we only pass a buf and buf length to
158958360b13Sdlg 	 * noise_remote_encrypt. Technically it would be possible to teach
159058360b13Sdlg 	 * noise_remote_encrypt about mbufs, but we would need to sort out the
159158360b13Sdlg 	 * p_encap_queue situation first.
159258360b13Sdlg 	 */
1593c0619985Sclaudio 	if ((mc = m_clget(NULL, M_NOWAIT, out_len + max_hdr)) == NULL)
159458360b13Sdlg 		goto error;
1595c0619985Sclaudio 	m_align(mc, out_len);
159658360b13Sdlg 
159758360b13Sdlg 	data = mtod(mc, struct wg_pkt_data *);
159858360b13Sdlg 	m_copydata(m, 0, m->m_pkthdr.len, data->buf);
159958360b13Sdlg 	bzero(data->buf + m->m_pkthdr.len, padding_len);
160058360b13Sdlg 	data->t = WG_PKT_DATA;
160158360b13Sdlg 
160258360b13Sdlg 	/*
160358360b13Sdlg 	 * Copy the flow hash from the inner packet to the outer packet, so
160458360b13Sdlg 	 * that fq_codel can property separate streams, rather than falling
160558360b13Sdlg 	 * back to random buckets.
160658360b13Sdlg 	 */
160758360b13Sdlg 	mc->m_pkthdr.ph_flowid = m->m_pkthdr.ph_flowid;
160858360b13Sdlg 
1609dbebf518Ssthen 	mc->m_pkthdr.pf.prio = m->m_pkthdr.pf.prio;
1610dbebf518Ssthen 
161158360b13Sdlg 	res = noise_remote_encrypt(&peer->p_remote, &data->r_idx, &nonce,
161258360b13Sdlg 				   data->buf, plaintext_len);
161358360b13Sdlg 	nonce = htole64(nonce); /* Wire format is little endian. */
161458360b13Sdlg 	memcpy(data->nonce, &nonce, sizeof(data->nonce));
161558360b13Sdlg 
161658360b13Sdlg 	if (__predict_false(res == EINVAL)) {
161758360b13Sdlg 		m_freem(mc);
161858360b13Sdlg 		goto error;
161958360b13Sdlg 	} else if (__predict_false(res == ESTALE)) {
162058360b13Sdlg 		wg_timers_event_want_initiation(&peer->p_timers);
162158360b13Sdlg 	} else if (__predict_false(res != 0)) {
16224123b6a7Sderaadt 		panic("unexpected result: %d", res);
162358360b13Sdlg 	}
162458360b13Sdlg 
162558360b13Sdlg 	/* A packet with length 0 is a keepalive packet */
162658360b13Sdlg 	if (__predict_false(m->m_pkthdr.len == 0))
1627*f08653c5Smvs 		WGPRINTF(LOG_DEBUG, sc, &peer->p_endpoint_mtx, "Sending "
1628*f08653c5Smvs 		    "keepalive packet to peer %llu (%s)\n", peer->p_id,
1629*f08653c5Smvs 		    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa, ipaddr,
1630*f08653c5Smvs 		        sizeof(ipaddr)));
163158360b13Sdlg 
163258360b13Sdlg 	mc->m_pkthdr.ph_loopcnt = m->m_pkthdr.ph_loopcnt;
163358360b13Sdlg 	mc->m_flags &= ~(M_MCAST | M_BCAST);
1634c0619985Sclaudio 	mc->m_pkthdr.len = mc->m_len = out_len;
163558360b13Sdlg 
163658360b13Sdlg 	/*
163758360b13Sdlg 	 * We would count ifc_opackets, ifc_obytes of m here, except if_snd
163858360b13Sdlg 	 * already does that for us, so no need to worry about it.
163958360b13Sdlg 	counters_pkt(sc->sc_if.if_counters, ifc_opackets, ifc_obytes,
164058360b13Sdlg 	    m->m_pkthdr.len);
164158360b13Sdlg 	 */
164258360b13Sdlg 	wg_peer_counters_add(peer, mc->m_pkthdr.len, 0);
164358360b13Sdlg 
164458360b13Sdlg 	t->t_mbuf = mc;
164558360b13Sdlg error:
164658360b13Sdlg 	t->t_done = 1;
164758360b13Sdlg 	task_add(net_tq(sc->sc_if.if_index), &peer->p_deliver_out);
164858360b13Sdlg }
164958360b13Sdlg 
165058360b13Sdlg void
165158360b13Sdlg wg_decap(struct wg_softc *sc, struct mbuf *m)
165258360b13Sdlg {
165358360b13Sdlg 	int			 res, len;
165458360b13Sdlg 	struct ip		*ip;
165558360b13Sdlg 	struct ip6_hdr		*ip6;
165658360b13Sdlg 	struct wg_pkt_data	*data;
165758360b13Sdlg 	struct wg_peer		*peer, *allowed_peer;
165858360b13Sdlg 	struct wg_tag		*t;
165958360b13Sdlg 	size_t			 payload_len;
166058360b13Sdlg 	uint64_t		 nonce;
1661*f08653c5Smvs 	char			 ipaddr[INET6_ADDRSTRLEN];
166258360b13Sdlg 
166358360b13Sdlg 	t = wg_tag_get(m);
166458360b13Sdlg 	peer = t->t_peer;
166558360b13Sdlg 
166658360b13Sdlg 	/*
166758360b13Sdlg 	 * Likewise to wg_encap, we pass a buf and buf length to
166858360b13Sdlg 	 * noise_remote_decrypt. Again, possible to teach it about mbufs
166958360b13Sdlg 	 * but need to get over the p_decap_queue situation first. However,
167058360b13Sdlg 	 * we do not need to allocate a new mbuf as the decrypted packet is
167158360b13Sdlg 	 * strictly smaller than encrypted. We just set t_mbuf to m and
167258360b13Sdlg 	 * wg_deliver_in knows how to deal with that.
167358360b13Sdlg 	 */
167458360b13Sdlg 	data = mtod(m, struct wg_pkt_data *);
167558360b13Sdlg 	payload_len = m->m_pkthdr.len - sizeof(struct wg_pkt_data);
167658360b13Sdlg 	memcpy(&nonce, data->nonce, sizeof(nonce));
167758360b13Sdlg 	nonce = le64toh(nonce); /* Wire format is little endian. */
167858360b13Sdlg 	res = noise_remote_decrypt(&peer->p_remote, data->r_idx, nonce,
167958360b13Sdlg 				   data->buf, payload_len);
168058360b13Sdlg 
168158360b13Sdlg 	if (__predict_false(res == EINVAL)) {
168258360b13Sdlg 		goto error;
168358360b13Sdlg 	} else if (__predict_false(res == ECONNRESET)) {
168458360b13Sdlg 		wg_timers_event_handshake_complete(&peer->p_timers);
168558360b13Sdlg 	} else if (__predict_false(res == ESTALE)) {
168658360b13Sdlg 		wg_timers_event_want_initiation(&peer->p_timers);
168758360b13Sdlg 	} else if (__predict_false(res != 0)) {
16884123b6a7Sderaadt 		panic("unexpected response: %d", res);
168958360b13Sdlg 	}
169058360b13Sdlg 
169158360b13Sdlg 	wg_peer_set_endpoint_from_tag(peer, t);
169258360b13Sdlg 
169358360b13Sdlg 	wg_peer_counters_add(peer, 0, m->m_pkthdr.len);
169458360b13Sdlg 
169558360b13Sdlg 	m_adj(m, sizeof(struct wg_pkt_data));
169658360b13Sdlg 	m_adj(m, -NOISE_AUTHTAG_LEN);
169758360b13Sdlg 
169858360b13Sdlg 	counters_pkt(sc->sc_if.if_counters, ifc_ipackets, ifc_ibytes,
169958360b13Sdlg 	    m->m_pkthdr.len);
170058360b13Sdlg 
170158360b13Sdlg 	/* A packet with length 0 is a keepalive packet */
170258360b13Sdlg 	if (__predict_false(m->m_pkthdr.len == 0)) {
1703*f08653c5Smvs 		WGPRINTF(LOG_DEBUG, sc, &peer->p_endpoint_mtx, "Receiving "
1704*f08653c5Smvs 		    "keepalive packet from peer %llu (%s)\n", peer->p_id,
1705*f08653c5Smvs 		    sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa,
1706*f08653c5Smvs 		        ipaddr, sizeof(ipaddr)));
170758360b13Sdlg 		goto done;
170858360b13Sdlg 	}
170958360b13Sdlg 
171058360b13Sdlg 	/*
171158360b13Sdlg 	 * We can let the network stack handle the intricate validation of the
171258360b13Sdlg 	 * IP header, we just worry about the sizeof and the version, so we can
171358360b13Sdlg 	 * read the source address in wg_aip_lookup.
171458360b13Sdlg 	 *
1715df8d9afdSjsg 	 * We also need to trim the packet, as it was likely padded before
171658360b13Sdlg 	 * encryption. While we could drop it here, it will be more helpful to
171758360b13Sdlg 	 * pass it to bpf_mtap and use the counters that people are expecting
171858360b13Sdlg 	 * in ipv4_input and ipv6_input. We can rely on ipv4_input and
171958360b13Sdlg 	 * ipv6_input to properly validate the headers.
172058360b13Sdlg 	 */
172158360b13Sdlg 	ip = mtod(m, struct ip *);
172258360b13Sdlg 	ip6 = mtod(m, struct ip6_hdr *);
172358360b13Sdlg 
172458360b13Sdlg 	if (m->m_pkthdr.len >= sizeof(struct ip) && ip->ip_v == IPVERSION) {
172558360b13Sdlg 		m->m_pkthdr.ph_family = AF_INET;
172658360b13Sdlg 
172758360b13Sdlg 		len = ntohs(ip->ip_len);
172858360b13Sdlg 		if (len >= sizeof(struct ip) && len < m->m_pkthdr.len)
172958360b13Sdlg 			m_adj(m, len - m->m_pkthdr.len);
173058360b13Sdlg 
173158360b13Sdlg 		allowed_peer = wg_aip_lookup(sc->sc_aip4, &ip->ip_src);
173258360b13Sdlg #ifdef INET6
173358360b13Sdlg 	} else if (m->m_pkthdr.len >= sizeof(struct ip6_hdr) &&
173458360b13Sdlg 	    (ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) {
173558360b13Sdlg 		m->m_pkthdr.ph_family = AF_INET6;
173658360b13Sdlg 
173758360b13Sdlg 		len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr);
173858360b13Sdlg 		if (len < m->m_pkthdr.len)
173958360b13Sdlg 			m_adj(m, len - m->m_pkthdr.len);
174058360b13Sdlg 
174158360b13Sdlg 		allowed_peer = wg_aip_lookup(sc->sc_aip6, &ip6->ip6_src);
174258360b13Sdlg #endif
174358360b13Sdlg 	} else {
1744*f08653c5Smvs 		WGPRINTF(LOG_WARNING, sc, &peer->p_endpoint_mtx, "Packet "
1745*f08653c5Smvs 		    "is neither IPv4 nor IPv6 from peer %llu (%s)\n",
1746*f08653c5Smvs 		    peer->p_id, sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa,
1747*f08653c5Smvs 		        ipaddr, sizeof(ipaddr)));
174858360b13Sdlg 		goto error;
174958360b13Sdlg 	}
175058360b13Sdlg 
175158360b13Sdlg 	if (__predict_false(peer != allowed_peer)) {
1752*f08653c5Smvs 		WGPRINTF(LOG_WARNING, sc, &peer->p_endpoint_mtx, "Packet "
1753*f08653c5Smvs                     "has unallowed source IP from peer %llu (%s)\n",
1754*f08653c5Smvs                     peer->p_id, sockaddr_ntop(&peer->p_endpoint.e_remote.r_sa,
1755*f08653c5Smvs                         ipaddr, sizeof(ipaddr)));
175658360b13Sdlg 		goto error;
175758360b13Sdlg 	}
175858360b13Sdlg 
17598069430eSprocter 	/* tunneled packet was not offloaded */
17608069430eSprocter 	m->m_pkthdr.csum_flags = 0;
176158360b13Sdlg 
176258360b13Sdlg 	m->m_pkthdr.ph_ifidx = sc->sc_if.if_index;
176358360b13Sdlg 	m->m_pkthdr.ph_rtableid = sc->sc_if.if_rdomain;
176458360b13Sdlg 	m->m_flags &= ~(M_MCAST | M_BCAST);
176589fdc1a9Skn #if NPF > 0
176658360b13Sdlg 	pf_pkt_addr_changed(m);
176789fdc1a9Skn #endif /* NPF > 0 */
176858360b13Sdlg 
176958360b13Sdlg done:
177058360b13Sdlg 	t->t_mbuf = m;
177158360b13Sdlg error:
177258360b13Sdlg 	t->t_done = 1;
177358360b13Sdlg 	task_add(net_tq(sc->sc_if.if_index), &peer->p_deliver_in);
177458360b13Sdlg }
177558360b13Sdlg 
177658360b13Sdlg void
177758360b13Sdlg wg_encap_worker(void *_sc)
177858360b13Sdlg {
177958360b13Sdlg 	struct mbuf *m;
178058360b13Sdlg 	struct wg_softc *sc = _sc;
178158360b13Sdlg 	while ((m = wg_ring_dequeue(&sc->sc_encap_ring)) != NULL)
178258360b13Sdlg 		wg_encap(sc, m);
178358360b13Sdlg }
178458360b13Sdlg 
178558360b13Sdlg void
178658360b13Sdlg wg_decap_worker(void *_sc)
178758360b13Sdlg {
178858360b13Sdlg 	struct mbuf *m;
178958360b13Sdlg 	struct wg_softc *sc = _sc;
179058360b13Sdlg 	while ((m = wg_ring_dequeue(&sc->sc_decap_ring)) != NULL)
179158360b13Sdlg 		wg_decap(sc, m);
179258360b13Sdlg }
179358360b13Sdlg 
179458360b13Sdlg void
179558360b13Sdlg wg_deliver_out(void *_peer)
179658360b13Sdlg {
179758360b13Sdlg 	struct wg_peer		*peer = _peer;
179858360b13Sdlg 	struct wg_softc		*sc = peer->p_sc;
179958360b13Sdlg 	struct wg_endpoint	 endpoint;
180058360b13Sdlg 	struct wg_tag		*t;
180158360b13Sdlg 	struct mbuf		*m;
180258360b13Sdlg 	int			 ret;
180358360b13Sdlg 
180458360b13Sdlg 	wg_peer_get_endpoint(peer, &endpoint);
180558360b13Sdlg 
180658360b13Sdlg 	while ((m = wg_queue_dequeue(&peer->p_encap_queue, &t)) != NULL) {
180758360b13Sdlg 		/* t_mbuf will contain the encrypted packet */
180858360b13Sdlg 		if (t->t_mbuf == NULL){
180958360b13Sdlg 			counters_inc(sc->sc_if.if_counters, ifc_oerrors);
181058360b13Sdlg 			m_freem(m);
181158360b13Sdlg 			continue;
181258360b13Sdlg 		}
181358360b13Sdlg 
181458360b13Sdlg 		ret = wg_send(sc, &endpoint, t->t_mbuf);
181558360b13Sdlg 
181658360b13Sdlg 		if (ret == 0) {
181758360b13Sdlg 			wg_timers_event_any_authenticated_packet_traversal(
181858360b13Sdlg 			    &peer->p_timers);
181958360b13Sdlg 			wg_timers_event_any_authenticated_packet_sent(
182058360b13Sdlg 			    &peer->p_timers);
182158360b13Sdlg 
182258360b13Sdlg 			if (m->m_pkthdr.len != 0)
182358360b13Sdlg 				wg_timers_event_data_sent(&peer->p_timers);
182458360b13Sdlg 		} else if (ret == EADDRNOTAVAIL) {
182558360b13Sdlg 			wg_peer_clear_src(peer);
182658360b13Sdlg 			wg_peer_get_endpoint(peer, &endpoint);
182758360b13Sdlg 		}
182858360b13Sdlg 
182958360b13Sdlg 		m_freem(m);
183058360b13Sdlg 	}
183158360b13Sdlg }
183258360b13Sdlg 
183358360b13Sdlg void
183458360b13Sdlg wg_deliver_in(void *_peer)
183558360b13Sdlg {
183658360b13Sdlg 	struct wg_peer	*peer = _peer;
183758360b13Sdlg 	struct wg_softc	*sc = peer->p_sc;
183858360b13Sdlg 	struct wg_tag	*t;
183958360b13Sdlg 	struct mbuf	*m;
184058360b13Sdlg 
184158360b13Sdlg 	while ((m = wg_queue_dequeue(&peer->p_decap_queue, &t)) != NULL) {
184258360b13Sdlg 		/* t_mbuf will contain the decrypted packet */
184358360b13Sdlg 		if (t->t_mbuf == NULL) {
184458360b13Sdlg 			counters_inc(sc->sc_if.if_counters, ifc_ierrors);
184558360b13Sdlg 			m_freem(m);
184658360b13Sdlg 			continue;
184758360b13Sdlg 		}
184858360b13Sdlg 
184958360b13Sdlg 		/* From here on m == t->t_mbuf */
185058360b13Sdlg 		KASSERT(m == t->t_mbuf);
185158360b13Sdlg 
185258360b13Sdlg 		wg_timers_event_any_authenticated_packet_received(
185358360b13Sdlg 		    &peer->p_timers);
185458360b13Sdlg 		wg_timers_event_any_authenticated_packet_traversal(
185558360b13Sdlg 		    &peer->p_timers);
185658360b13Sdlg 
185758360b13Sdlg 		if (m->m_pkthdr.len == 0) {
185858360b13Sdlg 			m_freem(m);
185958360b13Sdlg 			continue;
186058360b13Sdlg 		}
186158360b13Sdlg 
186258360b13Sdlg #if NBPFILTER > 0
186358360b13Sdlg 		if (sc->sc_if.if_bpf != NULL)
186458360b13Sdlg 			bpf_mtap_af(sc->sc_if.if_bpf,
186558360b13Sdlg 			    m->m_pkthdr.ph_family, m, BPF_DIRECTION_IN);
186658360b13Sdlg #endif
186758360b13Sdlg 
186858360b13Sdlg 		NET_LOCK();
186958360b13Sdlg 		if (m->m_pkthdr.ph_family == AF_INET)
187058360b13Sdlg 			ipv4_input(&sc->sc_if, m);
187158360b13Sdlg #ifdef INET6
187258360b13Sdlg 		else if (m->m_pkthdr.ph_family == AF_INET6)
187358360b13Sdlg 			ipv6_input(&sc->sc_if, m);
187458360b13Sdlg #endif
187558360b13Sdlg 		else
187658360b13Sdlg 			panic("invalid ph_family");
187758360b13Sdlg 		NET_UNLOCK();
187858360b13Sdlg 
187958360b13Sdlg 		wg_timers_event_data_received(&peer->p_timers);
188058360b13Sdlg 	}
188158360b13Sdlg }
188258360b13Sdlg 
188358360b13Sdlg int
188458360b13Sdlg wg_queue_in(struct wg_softc *sc, struct wg_peer *peer, struct mbuf *m)
188558360b13Sdlg {
188658360b13Sdlg 	struct wg_ring		*parallel = &sc->sc_decap_ring;
188758360b13Sdlg 	struct wg_queue		*serial = &peer->p_decap_queue;
188858360b13Sdlg 	struct wg_tag		*t;
188958360b13Sdlg 
189058360b13Sdlg 	mtx_enter(&serial->q_mtx);
189158360b13Sdlg 	if (serial->q_list.ml_len < MAX_QUEUED_PKT) {
189258360b13Sdlg 		ml_enqueue(&serial->q_list, m);
189358360b13Sdlg 		mtx_leave(&serial->q_mtx);
189458360b13Sdlg 	} else {
189558360b13Sdlg 		mtx_leave(&serial->q_mtx);
189658360b13Sdlg 		m_freem(m);
189758360b13Sdlg 		return ENOBUFS;
189858360b13Sdlg 	}
189958360b13Sdlg 
190058360b13Sdlg 	mtx_enter(&parallel->r_mtx);
190158360b13Sdlg 	if (parallel->r_tail - parallel->r_head < MAX_QUEUED_PKT) {
190258360b13Sdlg 		parallel->r_buf[parallel->r_tail & MAX_QUEUED_PKT_MASK] = m;
190358360b13Sdlg 		parallel->r_tail++;
190458360b13Sdlg 		mtx_leave(&parallel->r_mtx);
190558360b13Sdlg 	} else {
190658360b13Sdlg 		mtx_leave(&parallel->r_mtx);
190758360b13Sdlg 		t = wg_tag_get(m);
190858360b13Sdlg 		t->t_done = 1;
190958360b13Sdlg 		return ENOBUFS;
191058360b13Sdlg 	}
191158360b13Sdlg 
191258360b13Sdlg 	return 0;
191358360b13Sdlg }
191458360b13Sdlg 
191558360b13Sdlg void
191658360b13Sdlg wg_queue_out(struct wg_softc *sc, struct wg_peer *peer)
191758360b13Sdlg {
191858360b13Sdlg 	struct wg_ring		*parallel = &sc->sc_encap_ring;
191958360b13Sdlg 	struct wg_queue		*serial = &peer->p_encap_queue;
192058360b13Sdlg 	struct mbuf_list 	 ml, ml_free;
192158360b13Sdlg 	struct mbuf		*m;
192258360b13Sdlg 	struct wg_tag		*t;
192358360b13Sdlg 	int			 dropped;
192458360b13Sdlg 
192558360b13Sdlg 	/*
192658360b13Sdlg 	 * We delist all staged packets and then add them to the queues. This
19278569c6f6Stobhe 	 * can race with wg_qstart when called from wg_send_keepalive, however
19288569c6f6Stobhe 	 * wg_qstart will not race as it is serialised.
192958360b13Sdlg 	 */
193058360b13Sdlg 	mq_delist(&peer->p_stage_queue, &ml);
193158360b13Sdlg 	ml_init(&ml_free);
193258360b13Sdlg 
193358360b13Sdlg 	while ((m = ml_dequeue(&ml)) != NULL) {
193458360b13Sdlg 		mtx_enter(&serial->q_mtx);
193558360b13Sdlg 		if (serial->q_list.ml_len < MAX_QUEUED_PKT) {
193658360b13Sdlg 			ml_enqueue(&serial->q_list, m);
193758360b13Sdlg 			mtx_leave(&serial->q_mtx);
193858360b13Sdlg 		} else {
193958360b13Sdlg 			mtx_leave(&serial->q_mtx);
194058360b13Sdlg 			ml_enqueue(&ml_free, m);
194158360b13Sdlg 			continue;
194258360b13Sdlg 		}
194358360b13Sdlg 
194458360b13Sdlg 		mtx_enter(&parallel->r_mtx);
194558360b13Sdlg 		if (parallel->r_tail - parallel->r_head < MAX_QUEUED_PKT) {
194658360b13Sdlg 			parallel->r_buf[parallel->r_tail & MAX_QUEUED_PKT_MASK] = m;
194758360b13Sdlg 			parallel->r_tail++;
194858360b13Sdlg 			mtx_leave(&parallel->r_mtx);
194958360b13Sdlg 		} else {
195058360b13Sdlg 			mtx_leave(&parallel->r_mtx);
195158360b13Sdlg 			t = wg_tag_get(m);
195258360b13Sdlg 			t->t_done = 1;
195358360b13Sdlg 		}
195458360b13Sdlg 	}
195558360b13Sdlg 
195658360b13Sdlg 	if ((dropped = ml_purge(&ml_free)) > 0)
195758360b13Sdlg 		counters_add(sc->sc_if.if_counters, ifc_oqdrops, dropped);
195858360b13Sdlg }
195958360b13Sdlg 
196058360b13Sdlg struct mbuf *
196158360b13Sdlg wg_ring_dequeue(struct wg_ring *r)
196258360b13Sdlg {
196358360b13Sdlg 	struct mbuf *m = NULL;
196458360b13Sdlg 	mtx_enter(&r->r_mtx);
196558360b13Sdlg 	if (r->r_head != r->r_tail) {
196658360b13Sdlg 		m = r->r_buf[r->r_head & MAX_QUEUED_PKT_MASK];
196758360b13Sdlg 		r->r_head++;
196858360b13Sdlg 	}
196958360b13Sdlg 	mtx_leave(&r->r_mtx);
197058360b13Sdlg 	return m;
197158360b13Sdlg }
197258360b13Sdlg 
197358360b13Sdlg struct mbuf *
197458360b13Sdlg wg_queue_dequeue(struct wg_queue *q, struct wg_tag **t)
197558360b13Sdlg {
197658360b13Sdlg 	struct mbuf *m;
197758360b13Sdlg 	mtx_enter(&q->q_mtx);
197858360b13Sdlg 	if ((m = q->q_list.ml_head) != NULL && (*t = wg_tag_get(m))->t_done)
197958360b13Sdlg 		ml_dequeue(&q->q_list);
198058360b13Sdlg 	else
198158360b13Sdlg 		m = NULL;
198258360b13Sdlg 	mtx_leave(&q->q_mtx);
198358360b13Sdlg 	return m;
198458360b13Sdlg }
198558360b13Sdlg 
198658360b13Sdlg size_t
198758360b13Sdlg wg_queue_len(struct wg_queue *q)
198858360b13Sdlg {
198958360b13Sdlg 	size_t len;
199058360b13Sdlg 	mtx_enter(&q->q_mtx);
199158360b13Sdlg 	len = q->q_list.ml_len;
199258360b13Sdlg 	mtx_leave(&q->q_mtx);
199358360b13Sdlg 	return len;
199458360b13Sdlg }
199558360b13Sdlg 
199658360b13Sdlg struct noise_remote *
199758360b13Sdlg wg_remote_get(void *_sc, uint8_t public[NOISE_PUBLIC_KEY_LEN])
199858360b13Sdlg {
199958360b13Sdlg 	struct wg_peer	*peer;
200058360b13Sdlg 	struct wg_softc	*sc = _sc;
200158360b13Sdlg 	if ((peer = wg_peer_lookup(sc, public)) == NULL)
200258360b13Sdlg 		return NULL;
200358360b13Sdlg 	return &peer->p_remote;
200458360b13Sdlg }
200558360b13Sdlg 
200658360b13Sdlg uint32_t
200758360b13Sdlg wg_index_set(void *_sc, struct noise_remote *remote)
200858360b13Sdlg {
200958360b13Sdlg 	struct wg_peer	*peer;
201058360b13Sdlg 	struct wg_softc	*sc = _sc;
201158360b13Sdlg 	struct wg_index *index, *iter;
201258360b13Sdlg 	uint32_t	 key;
201358360b13Sdlg 
201458360b13Sdlg 	/*
201558360b13Sdlg 	 * We can modify this without a lock as wg_index_set, wg_index_drop are
201658360b13Sdlg 	 * guaranteed to be serialised (per remote).
201758360b13Sdlg 	 */
201858360b13Sdlg 	peer = CONTAINER_OF(remote, struct wg_peer, p_remote);
201958360b13Sdlg 	index = SLIST_FIRST(&peer->p_unused_index);
202058360b13Sdlg 	KASSERT(index != NULL);
202158360b13Sdlg 	SLIST_REMOVE_HEAD(&peer->p_unused_index, i_unused_entry);
202258360b13Sdlg 
202358360b13Sdlg 	index->i_value = remote;
202458360b13Sdlg 
202558360b13Sdlg 	mtx_enter(&sc->sc_index_mtx);
202658360b13Sdlg assign_id:
202758360b13Sdlg 	key = index->i_key = arc4random();
202858360b13Sdlg 	key &= sc->sc_index_mask;
202958360b13Sdlg 	LIST_FOREACH(iter, &sc->sc_index[key], i_entry)
203058360b13Sdlg 		if (iter->i_key == index->i_key)
203158360b13Sdlg 			goto assign_id;
203258360b13Sdlg 
203358360b13Sdlg 	LIST_INSERT_HEAD(&sc->sc_index[key], index, i_entry);
203458360b13Sdlg 
203558360b13Sdlg 	mtx_leave(&sc->sc_index_mtx);
203658360b13Sdlg 
203758360b13Sdlg 	/* Likewise, no need to lock for index here. */
203858360b13Sdlg 	return index->i_key;
203958360b13Sdlg }
204058360b13Sdlg 
204158360b13Sdlg struct noise_remote *
204258360b13Sdlg wg_index_get(void *_sc, uint32_t key0)
204358360b13Sdlg {
204458360b13Sdlg 	struct wg_softc		*sc = _sc;
204558360b13Sdlg 	struct wg_index		*iter;
204658360b13Sdlg 	struct noise_remote	*remote = NULL;
204758360b13Sdlg 	uint32_t		 key = key0 & sc->sc_index_mask;
204858360b13Sdlg 
204958360b13Sdlg 	mtx_enter(&sc->sc_index_mtx);
205058360b13Sdlg 	LIST_FOREACH(iter, &sc->sc_index[key], i_entry)
205158360b13Sdlg 		if (iter->i_key == key0) {
205258360b13Sdlg 			remote = iter->i_value;
205358360b13Sdlg 			break;
205458360b13Sdlg 		}
205558360b13Sdlg 	mtx_leave(&sc->sc_index_mtx);
205658360b13Sdlg 	return remote;
205758360b13Sdlg }
205858360b13Sdlg 
205958360b13Sdlg void
206058360b13Sdlg wg_index_drop(void *_sc, uint32_t key0)
206158360b13Sdlg {
206258360b13Sdlg 	struct wg_softc	*sc = _sc;
206358360b13Sdlg 	struct wg_index	*iter;
206458360b13Sdlg 	struct wg_peer	*peer = NULL;
206558360b13Sdlg 	uint32_t	 key = key0 & sc->sc_index_mask;
206658360b13Sdlg 
206758360b13Sdlg 	mtx_enter(&sc->sc_index_mtx);
206858360b13Sdlg 	LIST_FOREACH(iter, &sc->sc_index[key], i_entry)
206958360b13Sdlg 		if (iter->i_key == key0) {
207058360b13Sdlg 			LIST_REMOVE(iter, i_entry);
207158360b13Sdlg 			break;
207258360b13Sdlg 		}
207358360b13Sdlg 	mtx_leave(&sc->sc_index_mtx);
207458360b13Sdlg 
207558360b13Sdlg 	/* We expect a peer */
207658360b13Sdlg 	peer = CONTAINER_OF(iter->i_value, struct wg_peer, p_remote);
207758360b13Sdlg 	KASSERT(peer != NULL);
207858360b13Sdlg 	SLIST_INSERT_HEAD(&peer->p_unused_index, iter, i_unused_entry);
207958360b13Sdlg }
208058360b13Sdlg 
208158360b13Sdlg struct mbuf *
208258360b13Sdlg wg_input(void *_sc, struct mbuf *m, struct ip *ip, struct ip6_hdr *ip6,
208358360b13Sdlg     void *_uh, int hlen)
208458360b13Sdlg {
208558360b13Sdlg 	struct wg_pkt_data	*data;
208658360b13Sdlg 	struct noise_remote	*remote;
208758360b13Sdlg 	struct wg_tag		*t;
208858360b13Sdlg 	struct wg_softc		*sc = _sc;
208958360b13Sdlg 	struct udphdr		*uh = _uh;
2090*f08653c5Smvs 	char			 ipaddr[INET6_ADDRSTRLEN];
209158360b13Sdlg 
209258360b13Sdlg 	NET_ASSERT_LOCKED();
209358360b13Sdlg 
209458360b13Sdlg 	if ((t = wg_tag_get(m)) == NULL) {
209558360b13Sdlg 		m_freem(m);
209658360b13Sdlg 		return NULL;
209758360b13Sdlg 	}
209858360b13Sdlg 
209958360b13Sdlg 	if (ip != NULL) {
210058360b13Sdlg 		t->t_endpoint.e_remote.r_sa.sa_len = sizeof(struct sockaddr_in);
210158360b13Sdlg 		t->t_endpoint.e_remote.r_sa.sa_family = AF_INET;
210258360b13Sdlg 		t->t_endpoint.e_remote.r_sin.sin_port = uh->uh_sport;
210358360b13Sdlg 		t->t_endpoint.e_remote.r_sin.sin_addr = ip->ip_src;
210458360b13Sdlg 		t->t_endpoint.e_local.l_in = ip->ip_dst;
210558360b13Sdlg #ifdef INET6
210658360b13Sdlg 	} else if (ip6 != NULL) {
210758360b13Sdlg 		t->t_endpoint.e_remote.r_sa.sa_len = sizeof(struct sockaddr_in6);
210858360b13Sdlg 		t->t_endpoint.e_remote.r_sa.sa_family = AF_INET6;
210958360b13Sdlg 		t->t_endpoint.e_remote.r_sin6.sin6_port = uh->uh_sport;
211058360b13Sdlg 		t->t_endpoint.e_remote.r_sin6.sin6_addr = ip6->ip6_src;
211158360b13Sdlg 		t->t_endpoint.e_local.l_in6 = ip6->ip6_dst;
211258360b13Sdlg #endif
211358360b13Sdlg 	} else {
211458360b13Sdlg 		m_freem(m);
211558360b13Sdlg 		return NULL;
211658360b13Sdlg 	}
211758360b13Sdlg 
211858360b13Sdlg 	/* m has a IP/IPv6 header of hlen length, we don't need it anymore. */
211958360b13Sdlg 	m_adj(m, hlen);
212058360b13Sdlg 
2121dc736ae2Stb 	/*
2122dc736ae2Stb 	 * Ensure mbuf is contiguous over full length of packet. This is done
2123c04adf2dSsthen 	 * so we can directly read the handshake values in wg_handshake, and so
2124dc736ae2Stb 	 * we can decrypt a transport packet by passing a single buffer to
2125dc736ae2Stb 	 * noise_remote_decrypt in wg_decap.
2126dc736ae2Stb 	 */
2127dc736ae2Stb 	if ((m = m_pullup(m, m->m_pkthdr.len)) == NULL)
212858360b13Sdlg 		return NULL;
212958360b13Sdlg 
213058360b13Sdlg 	if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) &&
213158360b13Sdlg 		*mtod(m, uint32_t *) == WG_PKT_INITIATION) ||
213258360b13Sdlg 	    (m->m_pkthdr.len == sizeof(struct wg_pkt_response) &&
213358360b13Sdlg 		*mtod(m, uint32_t *) == WG_PKT_RESPONSE) ||
213458360b13Sdlg 	    (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) &&
213558360b13Sdlg 		*mtod(m, uint32_t *) == WG_PKT_COOKIE)) {
213658360b13Sdlg 
213758360b13Sdlg 		if (mq_enqueue(&sc->sc_handshake_queue, m) != 0)
2138*f08653c5Smvs 			WGPRINTF(LOG_DEBUG, sc, NULL, "Dropping handshake"
2139*f08653c5Smvs 			    "packet from %s\n",
2140*f08653c5Smvs 			    sockaddr_ntop(&t->t_endpoint.e_remote.r_sa, ipaddr,
2141*f08653c5Smvs 			        sizeof(ipaddr)));
214258360b13Sdlg 		task_add(wg_handshake_taskq, &sc->sc_handshake);
214358360b13Sdlg 
214458360b13Sdlg 	} else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) +
214558360b13Sdlg 	    NOISE_AUTHTAG_LEN && *mtod(m, uint32_t *) == WG_PKT_DATA) {
214658360b13Sdlg 
214758360b13Sdlg 		data = mtod(m, struct wg_pkt_data *);
214858360b13Sdlg 
214958360b13Sdlg 		if ((remote = wg_index_get(sc, data->r_idx)) != NULL) {
215058360b13Sdlg 			t->t_peer = CONTAINER_OF(remote, struct wg_peer,
215158360b13Sdlg 			    p_remote);
215258360b13Sdlg 			t->t_mbuf = NULL;
215358360b13Sdlg 			t->t_done = 0;
215458360b13Sdlg 
215558360b13Sdlg 			if (wg_queue_in(sc, t->t_peer, m) != 0)
215658360b13Sdlg 				counters_inc(sc->sc_if.if_counters,
215758360b13Sdlg 				    ifc_iqdrops);
215858360b13Sdlg 			task_add(wg_crypt_taskq, &sc->sc_decap);
215958360b13Sdlg 		} else {
216058360b13Sdlg 			counters_inc(sc->sc_if.if_counters, ifc_ierrors);
216158360b13Sdlg 			m_freem(m);
216258360b13Sdlg 		}
216358360b13Sdlg 	} else {
216458360b13Sdlg 		counters_inc(sc->sc_if.if_counters, ifc_ierrors);
216558360b13Sdlg 		m_freem(m);
216658360b13Sdlg 	}
216758360b13Sdlg 
216858360b13Sdlg 	return NULL;
216958360b13Sdlg }
217058360b13Sdlg 
217158360b13Sdlg void
21728569c6f6Stobhe wg_qstart(struct ifqueue *ifq)
217358360b13Sdlg {
21748569c6f6Stobhe 	struct ifnet		*ifp = ifq->ifq_if;
217558360b13Sdlg 	struct wg_softc		*sc = ifp->if_softc;
217658360b13Sdlg 	struct wg_peer		*peer;
217758360b13Sdlg 	struct wg_tag		*t;
217858360b13Sdlg 	struct mbuf		*m;
217958360b13Sdlg 	SLIST_HEAD(,wg_peer)	 start_list;
218058360b13Sdlg 
218158360b13Sdlg 	SLIST_INIT(&start_list);
218258360b13Sdlg 
218358360b13Sdlg 	/*
218458360b13Sdlg 	 * We should be OK to modify p_start_list, p_start_onlist in this
21858569c6f6Stobhe 	 * function as there should only be one ifp->if_qstart invoked at a
21868569c6f6Stobhe 	 * time.
218758360b13Sdlg 	 */
21888569c6f6Stobhe 	while ((m = ifq_dequeue(ifq)) != NULL) {
218958360b13Sdlg 		t = wg_tag_get(m);
219058360b13Sdlg 		peer = t->t_peer;
219158360b13Sdlg 		if (mq_push(&peer->p_stage_queue, m) != 0)
219258360b13Sdlg 			counters_inc(ifp->if_counters, ifc_oqdrops);
219358360b13Sdlg 		if (!peer->p_start_onlist) {
219458360b13Sdlg 			SLIST_INSERT_HEAD(&start_list, peer, p_start_list);
219558360b13Sdlg 			peer->p_start_onlist = 1;
219658360b13Sdlg 		}
219758360b13Sdlg 	}
219858360b13Sdlg 	SLIST_FOREACH(peer, &start_list, p_start_list) {
219958360b13Sdlg 		if (noise_remote_ready(&peer->p_remote) == 0)
220058360b13Sdlg 			wg_queue_out(sc, peer);
220158360b13Sdlg 		else
220258360b13Sdlg 			wg_timers_event_want_initiation(&peer->p_timers);
220358360b13Sdlg 		peer->p_start_onlist = 0;
220458360b13Sdlg 	}
220558360b13Sdlg 	task_add(wg_crypt_taskq, &sc->sc_encap);
220658360b13Sdlg }
220758360b13Sdlg 
220858360b13Sdlg int
220958360b13Sdlg wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
221058360b13Sdlg     struct rtentry *rt)
221158360b13Sdlg {
221258360b13Sdlg 	struct wg_softc	*sc = ifp->if_softc;
221358360b13Sdlg 	struct wg_peer	*peer;
221458360b13Sdlg 	struct wg_tag	*t;
221558360b13Sdlg 	int		 af, ret = EINVAL;
221658360b13Sdlg 
221758360b13Sdlg 	NET_ASSERT_LOCKED();
221858360b13Sdlg 
221958360b13Sdlg 	if ((t = wg_tag_get(m)) == NULL) {
222058360b13Sdlg 		ret = ENOBUFS;
222158360b13Sdlg 		goto error;
222258360b13Sdlg 	}
222358360b13Sdlg 
222458360b13Sdlg 	m->m_pkthdr.ph_family = sa->sa_family;
222558360b13Sdlg 	if (sa->sa_family == AF_INET) {
222658360b13Sdlg 		peer = wg_aip_lookup(sc->sc_aip4,
222758360b13Sdlg 		    &mtod(m, struct ip *)->ip_dst);
222858360b13Sdlg #ifdef INET6
222958360b13Sdlg 	} else if (sa->sa_family == AF_INET6) {
223058360b13Sdlg 		peer = wg_aip_lookup(sc->sc_aip6,
223158360b13Sdlg 		    &mtod(m, struct ip6_hdr *)->ip6_dst);
223258360b13Sdlg #endif
223358360b13Sdlg 	} else {
223458360b13Sdlg 		ret = EAFNOSUPPORT;
223558360b13Sdlg 		goto error;
223658360b13Sdlg 	}
223758360b13Sdlg 
223858360b13Sdlg #if NBPFILTER > 0
223958360b13Sdlg 	if (sc->sc_if.if_bpf)
224058360b13Sdlg 		bpf_mtap_af(sc->sc_if.if_bpf, sa->sa_family, m,
224158360b13Sdlg 		    BPF_DIRECTION_OUT);
224258360b13Sdlg #endif
224358360b13Sdlg 
224458360b13Sdlg 	if (peer == NULL) {
224558360b13Sdlg 		ret = ENETUNREACH;
224658360b13Sdlg 		goto error;
224758360b13Sdlg 	}
224858360b13Sdlg 
224958360b13Sdlg 	af = peer->p_endpoint.e_remote.r_sa.sa_family;
225058360b13Sdlg 	if (af != AF_INET && af != AF_INET6) {
2251*f08653c5Smvs 		WGPRINTF(LOG_DEBUG, sc, NULL, "No valid endpoint has been "
2252*f08653c5Smvs 		    "configured or discovered for peer %llu\n", peer->p_id);
225358360b13Sdlg 		ret = EDESTADDRREQ;
225458360b13Sdlg 		goto error;
225558360b13Sdlg 	}
225658360b13Sdlg 
225758360b13Sdlg 	if (m->m_pkthdr.ph_loopcnt++ > M_MAXLOOP) {
2258*f08653c5Smvs 		WGPRINTF(LOG_DEBUG, sc, NULL, "Packet looped\n");
225958360b13Sdlg 		ret = ELOOP;
226058360b13Sdlg 		goto error;
226158360b13Sdlg 	}
226258360b13Sdlg 
226358360b13Sdlg 	/*
226458360b13Sdlg 	 * As we hold a reference to peer in the mbuf, we can't handle a
226558360b13Sdlg 	 * delayed packet without doing some refcnting. If a peer is removed
226658360b13Sdlg 	 * while a delayed holds a reference, bad things will happen. For the
226758360b13Sdlg 	 * time being, delayed packets are unsupported. This may be fixed with
22688569c6f6Stobhe 	 * another aip_lookup in wg_qstart, or refcnting as mentioned before.
226958360b13Sdlg 	 */
227058360b13Sdlg 	if (m->m_pkthdr.pf.delay > 0) {
2271*f08653c5Smvs 		WGPRINTF(LOG_DEBUG, sc, NULL, "PF delay unsupported\n");
227258360b13Sdlg 		ret = EOPNOTSUPP;
227358360b13Sdlg 		goto error;
227458360b13Sdlg 	}
227558360b13Sdlg 
227658360b13Sdlg 	t->t_peer = peer;
227758360b13Sdlg 	t->t_mbuf = NULL;
227858360b13Sdlg 	t->t_done = 0;
227958360b13Sdlg 	t->t_mtu = ifp->if_mtu;
228058360b13Sdlg 
228158360b13Sdlg 	/*
228258360b13Sdlg 	 * We still have an issue with ifq that will count a packet that gets
22838569c6f6Stobhe 	 * dropped in wg_qstart, or not encrypted. These get counted as
228458360b13Sdlg 	 * ofails or oqdrops, so the packet gets counted twice.
228558360b13Sdlg 	 */
228658360b13Sdlg 	return if_enqueue(ifp, m);
228758360b13Sdlg error:
228858360b13Sdlg 	counters_inc(ifp->if_counters, ifc_oerrors);
228958360b13Sdlg 	m_freem(m);
229058360b13Sdlg 	return ret;
229158360b13Sdlg }
229258360b13Sdlg 
229358360b13Sdlg int
229458360b13Sdlg wg_ioctl_set(struct wg_softc *sc, struct wg_data_io *data)
229558360b13Sdlg {
229658360b13Sdlg 	struct wg_interface_io	*iface_p, iface_o;
229758360b13Sdlg 	struct wg_peer_io	*peer_p, peer_o;
229858360b13Sdlg 	struct wg_aip_io	*aip_p, aip_o;
229958360b13Sdlg 
230058360b13Sdlg 	struct wg_peer		*peer, *tpeer;
230158360b13Sdlg 	struct wg_aip		*aip, *taip;
230258360b13Sdlg 
230358360b13Sdlg 	in_port_t		 port;
230458360b13Sdlg 	int			 rtable;
230558360b13Sdlg 
230658360b13Sdlg 	uint8_t			 public[WG_KEY_SIZE], private[WG_KEY_SIZE];
230758360b13Sdlg 	size_t			 i, j;
230858360b13Sdlg 	int			 ret, has_identity;
230958360b13Sdlg 
231058360b13Sdlg 	if ((ret = suser(curproc)) != 0)
231158360b13Sdlg 		return ret;
231258360b13Sdlg 
231358360b13Sdlg 	rw_enter_write(&sc->sc_lock);
231458360b13Sdlg 
231558360b13Sdlg 	iface_p = data->wgd_interface;
231658360b13Sdlg 	if ((ret = copyin(iface_p, &iface_o, sizeof(iface_o))) != 0)
231758360b13Sdlg 		goto error;
231858360b13Sdlg 
231958360b13Sdlg 	if (iface_o.i_flags & WG_INTERFACE_REPLACE_PEERS)
232058360b13Sdlg 		TAILQ_FOREACH_SAFE(peer, &sc->sc_peer_seq, p_seq_entry, tpeer)
232158360b13Sdlg 			wg_peer_destroy(peer);
232258360b13Sdlg 
232358360b13Sdlg 	if (iface_o.i_flags & WG_INTERFACE_HAS_PRIVATE &&
232458360b13Sdlg 	    (noise_local_keys(&sc->sc_local, NULL, private) ||
232558360b13Sdlg 	     timingsafe_bcmp(private, iface_o.i_private, WG_KEY_SIZE))) {
232658360b13Sdlg 		if (curve25519_generate_public(public, iface_o.i_private)) {
232758360b13Sdlg 			if ((peer = wg_peer_lookup(sc, public)) != NULL)
232858360b13Sdlg 				wg_peer_destroy(peer);
232958360b13Sdlg 		}
233058360b13Sdlg 		noise_local_lock_identity(&sc->sc_local);
233158360b13Sdlg 		has_identity = noise_local_set_private(&sc->sc_local,
233258360b13Sdlg 						       iface_o.i_private);
233358360b13Sdlg 		TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) {
233458360b13Sdlg 			noise_remote_precompute(&peer->p_remote);
233558360b13Sdlg 			wg_timers_event_reset_handshake_last_sent(&peer->p_timers);
233658360b13Sdlg 			noise_remote_expire_current(&peer->p_remote);
233758360b13Sdlg 		}
233858360b13Sdlg 		cookie_checker_update(&sc->sc_cookie,
233958360b13Sdlg 				      has_identity == 0 ? public : NULL);
234058360b13Sdlg 		noise_local_unlock_identity(&sc->sc_local);
234158360b13Sdlg 	}
234258360b13Sdlg 
234358360b13Sdlg 	if (iface_o.i_flags & WG_INTERFACE_HAS_PORT)
234458360b13Sdlg 		port = htons(iface_o.i_port);
234558360b13Sdlg 	else
234658360b13Sdlg 		port = sc->sc_udp_port;
234758360b13Sdlg 
234858360b13Sdlg 	if (iface_o.i_flags & WG_INTERFACE_HAS_RTABLE)
234958360b13Sdlg 		rtable = iface_o.i_rtable;
235058360b13Sdlg 	else
235158360b13Sdlg 		rtable = sc->sc_udp_rtable;
235258360b13Sdlg 
235358360b13Sdlg 	if (port != sc->sc_udp_port || rtable != sc->sc_udp_rtable) {
235458360b13Sdlg 		TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry)
235558360b13Sdlg 			wg_peer_clear_src(peer);
235658360b13Sdlg 
235758360b13Sdlg 		if (sc->sc_if.if_flags & IFF_RUNNING)
235858360b13Sdlg 			if ((ret = wg_bind(sc, &port, &rtable)) != 0)
235958360b13Sdlg 				goto error;
236058360b13Sdlg 
236158360b13Sdlg 		sc->sc_udp_port = port;
236258360b13Sdlg 		sc->sc_udp_rtable = rtable;
236358360b13Sdlg 	}
236458360b13Sdlg 
236558360b13Sdlg 	peer_p = &iface_p->i_peers[0];
236658360b13Sdlg 	for (i = 0; i < iface_o.i_peers_count; i++) {
236758360b13Sdlg 		if ((ret = copyin(peer_p, &peer_o, sizeof(peer_o))) != 0)
236858360b13Sdlg 			goto error;
236958360b13Sdlg 
237058360b13Sdlg 		/* Peer must have public key */
237158360b13Sdlg 		if (!(peer_o.p_flags & WG_PEER_HAS_PUBLIC))
2372a86e8a3fSyasuoka 			goto next_peer;
237358360b13Sdlg 
237458360b13Sdlg 		/* 0 = latest protocol, 1 = this protocol */
237558360b13Sdlg 		if (peer_o.p_protocol_version != 0) {
237658360b13Sdlg 			if (peer_o.p_protocol_version > 1) {
237758360b13Sdlg 				ret = EPFNOSUPPORT;
237858360b13Sdlg 				goto error;
237958360b13Sdlg 			}
238058360b13Sdlg 		}
238158360b13Sdlg 
238258360b13Sdlg 		/* Get local public and check that peer key doesn't match */
238358360b13Sdlg 		if (noise_local_keys(&sc->sc_local, public, NULL) == 0 &&
238458360b13Sdlg 		    bcmp(public, peer_o.p_public, WG_KEY_SIZE) == 0)
2385a86e8a3fSyasuoka 			goto next_peer;
238658360b13Sdlg 
238758360b13Sdlg 		/* Lookup peer, or create if it doesn't exist */
238858360b13Sdlg 		if ((peer = wg_peer_lookup(sc, peer_o.p_public)) == NULL) {
238958360b13Sdlg 			/* If we want to delete, no need creating a new one.
239058360b13Sdlg 			 * Also, don't create a new one if we only want to
239158360b13Sdlg 			 * update. */
239258360b13Sdlg 			if (peer_o.p_flags & (WG_PEER_REMOVE|WG_PEER_UPDATE))
2393a86e8a3fSyasuoka 				goto next_peer;
239458360b13Sdlg 
239558360b13Sdlg 			if ((peer = wg_peer_create(sc,
239658360b13Sdlg 			    peer_o.p_public)) == NULL) {
239758360b13Sdlg 				ret = ENOMEM;
239858360b13Sdlg 				goto error;
239958360b13Sdlg 			}
240058360b13Sdlg 		}
240158360b13Sdlg 
240258360b13Sdlg 		/* Remove peer and continue if specified */
240358360b13Sdlg 		if (peer_o.p_flags & WG_PEER_REMOVE) {
240458360b13Sdlg 			wg_peer_destroy(peer);
2405a86e8a3fSyasuoka 			goto next_peer;
240658360b13Sdlg 		}
240758360b13Sdlg 
240858360b13Sdlg 		if (peer_o.p_flags & WG_PEER_HAS_ENDPOINT)
240958360b13Sdlg 			wg_peer_set_sockaddr(peer, &peer_o.p_sa);
241058360b13Sdlg 
241158360b13Sdlg 		if (peer_o.p_flags & WG_PEER_HAS_PSK)
241258360b13Sdlg 			noise_remote_set_psk(&peer->p_remote, peer_o.p_psk);
241358360b13Sdlg 
241458360b13Sdlg 		if (peer_o.p_flags & WG_PEER_HAS_PKA)
241558360b13Sdlg 			wg_timers_set_persistent_keepalive(&peer->p_timers,
241658360b13Sdlg 			    peer_o.p_pka);
241758360b13Sdlg 
241858360b13Sdlg 		if (peer_o.p_flags & WG_PEER_REPLACE_AIPS) {
241958360b13Sdlg 			LIST_FOREACH_SAFE(aip, &peer->p_aip, a_entry, taip) {
242058360b13Sdlg 				wg_aip_remove(sc, peer, &aip->a_data);
242158360b13Sdlg 			}
242258360b13Sdlg 		}
242358360b13Sdlg 
242411d9cfa5Skn 		if (peer_o.p_flags & WG_PEER_SET_DESCRIPTION)
242511d9cfa5Skn 			strlcpy(peer->p_description, peer_o.p_description,
242611d9cfa5Skn 			    IFDESCRSIZE);
242711d9cfa5Skn 
242858360b13Sdlg 		aip_p = &peer_p->p_aips[0];
242958360b13Sdlg 		for (j = 0; j < peer_o.p_aips_count; j++) {
243058360b13Sdlg 			if ((ret = copyin(aip_p, &aip_o, sizeof(aip_o))) != 0)
243158360b13Sdlg 				goto error;
243258360b13Sdlg 			ret = wg_aip_add(sc, peer, &aip_o);
243358360b13Sdlg 			if (ret != 0)
243458360b13Sdlg 				goto error;
243558360b13Sdlg 			aip_p++;
243658360b13Sdlg 		}
243758360b13Sdlg 
243858360b13Sdlg 		peer_p = (struct wg_peer_io *)aip_p;
2439a86e8a3fSyasuoka 		continue;
2440a86e8a3fSyasuoka next_peer:
2441a86e8a3fSyasuoka 		aip_p = &peer_p->p_aips[0];
2442a86e8a3fSyasuoka 		aip_p += peer_o.p_aips_count;
2443a86e8a3fSyasuoka 		peer_p = (struct wg_peer_io *)aip_p;
244458360b13Sdlg 	}
244558360b13Sdlg 
244658360b13Sdlg error:
244758360b13Sdlg 	rw_exit_write(&sc->sc_lock);
244858360b13Sdlg 	explicit_bzero(&iface_o, sizeof(iface_o));
244958360b13Sdlg 	explicit_bzero(&peer_o, sizeof(peer_o));
245058360b13Sdlg 	explicit_bzero(&aip_o, sizeof(aip_o));
245158360b13Sdlg 	explicit_bzero(public, sizeof(public));
245258360b13Sdlg 	explicit_bzero(private, sizeof(private));
245358360b13Sdlg 	return ret;
245458360b13Sdlg }
245558360b13Sdlg 
245658360b13Sdlg int
245758360b13Sdlg wg_ioctl_get(struct wg_softc *sc, struct wg_data_io *data)
245858360b13Sdlg {
245958360b13Sdlg 	struct wg_interface_io	*iface_p, iface_o;
246058360b13Sdlg 	struct wg_peer_io	*peer_p, peer_o;
246158360b13Sdlg 	struct wg_aip_io	*aip_p;
246258360b13Sdlg 
246358360b13Sdlg 	struct wg_peer		*peer;
246458360b13Sdlg 	struct wg_aip		*aip;
246558360b13Sdlg 
246658360b13Sdlg 	size_t			 size, peer_count, aip_count;
246758360b13Sdlg 	int			 ret = 0, is_suser = suser(curproc) == 0;
246858360b13Sdlg 
246958360b13Sdlg 	size = sizeof(struct wg_interface_io);
247058360b13Sdlg 	if (data->wgd_size < size && !is_suser)
247158360b13Sdlg 		goto ret_size;
247258360b13Sdlg 
247358360b13Sdlg 	iface_p = data->wgd_interface;
247458360b13Sdlg 	bzero(&iface_o, sizeof(iface_o));
247558360b13Sdlg 
247658360b13Sdlg 	rw_enter_read(&sc->sc_lock);
247758360b13Sdlg 
247858360b13Sdlg 	if (sc->sc_udp_port != 0) {
247958360b13Sdlg 		iface_o.i_port = ntohs(sc->sc_udp_port);
248058360b13Sdlg 		iface_o.i_flags |= WG_INTERFACE_HAS_PORT;
248158360b13Sdlg 	}
248258360b13Sdlg 
248358360b13Sdlg 	if (sc->sc_udp_rtable != 0) {
248458360b13Sdlg 		iface_o.i_rtable = sc->sc_udp_rtable;
248558360b13Sdlg 		iface_o.i_flags |= WG_INTERFACE_HAS_RTABLE;
248658360b13Sdlg 	}
248758360b13Sdlg 
248858360b13Sdlg 	if (!is_suser)
248958360b13Sdlg 		goto copy_out_iface;
249058360b13Sdlg 
249158360b13Sdlg 	if (noise_local_keys(&sc->sc_local, iface_o.i_public,
249258360b13Sdlg 	    iface_o.i_private) == 0) {
249358360b13Sdlg 		iface_o.i_flags |= WG_INTERFACE_HAS_PUBLIC;
249458360b13Sdlg 		iface_o.i_flags |= WG_INTERFACE_HAS_PRIVATE;
249558360b13Sdlg 	}
249658360b13Sdlg 
249758360b13Sdlg 	size += sizeof(struct wg_peer_io) * sc->sc_peer_num;
249858360b13Sdlg 	size += sizeof(struct wg_aip_io) * sc->sc_aip_num;
249958360b13Sdlg 	if (data->wgd_size < size)
250058360b13Sdlg 		goto unlock_and_ret_size;
250158360b13Sdlg 
250258360b13Sdlg 	peer_count = 0;
250358360b13Sdlg 	peer_p = &iface_p->i_peers[0];
250458360b13Sdlg 	TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) {
250558360b13Sdlg 		bzero(&peer_o, sizeof(peer_o));
250658360b13Sdlg 		peer_o.p_flags = WG_PEER_HAS_PUBLIC;
250758360b13Sdlg 		peer_o.p_protocol_version = 1;
250858360b13Sdlg 
250958360b13Sdlg 		if (noise_remote_keys(&peer->p_remote, peer_o.p_public,
251058360b13Sdlg 		    peer_o.p_psk) == 0)
251158360b13Sdlg 			peer_o.p_flags |= WG_PEER_HAS_PSK;
251258360b13Sdlg 
251358360b13Sdlg 		if (wg_timers_get_persistent_keepalive(&peer->p_timers,
251458360b13Sdlg 		    &peer_o.p_pka) == 0)
251558360b13Sdlg 			peer_o.p_flags |= WG_PEER_HAS_PKA;
251658360b13Sdlg 
251758360b13Sdlg 		if (wg_peer_get_sockaddr(peer, &peer_o.p_sa) == 0)
251858360b13Sdlg 			peer_o.p_flags |= WG_PEER_HAS_ENDPOINT;
251958360b13Sdlg 
252058360b13Sdlg 		mtx_enter(&peer->p_counters_mtx);
252158360b13Sdlg 		peer_o.p_txbytes = peer->p_counters_tx;
252258360b13Sdlg 		peer_o.p_rxbytes = peer->p_counters_rx;
252358360b13Sdlg 		mtx_leave(&peer->p_counters_mtx);
252458360b13Sdlg 
252558360b13Sdlg 		wg_timers_get_last_handshake(&peer->p_timers,
252658360b13Sdlg 		    &peer_o.p_last_handshake);
252758360b13Sdlg 
252858360b13Sdlg 		aip_count = 0;
252958360b13Sdlg 		aip_p = &peer_p->p_aips[0];
253058360b13Sdlg 		LIST_FOREACH(aip, &peer->p_aip, a_entry) {
253158360b13Sdlg 			if ((ret = copyout(&aip->a_data, aip_p, sizeof(*aip_p))) != 0)
253258360b13Sdlg 				goto unlock_and_ret_size;
253358360b13Sdlg 			aip_p++;
253458360b13Sdlg 			aip_count++;
253558360b13Sdlg 		}
253658360b13Sdlg 		peer_o.p_aips_count = aip_count;
253758360b13Sdlg 
253811d9cfa5Skn 		strlcpy(peer_o.p_description, peer->p_description, IFDESCRSIZE);
253911d9cfa5Skn 
254058360b13Sdlg 		if ((ret = copyout(&peer_o, peer_p, sizeof(peer_o))) != 0)
254158360b13Sdlg 			goto unlock_and_ret_size;
254258360b13Sdlg 
254358360b13Sdlg 		peer_p = (struct wg_peer_io *)aip_p;
254458360b13Sdlg 		peer_count++;
254558360b13Sdlg 	}
254658360b13Sdlg 	iface_o.i_peers_count = peer_count;
254758360b13Sdlg 
254858360b13Sdlg copy_out_iface:
254958360b13Sdlg 	ret = copyout(&iface_o, iface_p, sizeof(iface_o));
255058360b13Sdlg unlock_and_ret_size:
255158360b13Sdlg 	rw_exit_read(&sc->sc_lock);
255258360b13Sdlg 	explicit_bzero(&iface_o, sizeof(iface_o));
255358360b13Sdlg 	explicit_bzero(&peer_o, sizeof(peer_o));
255458360b13Sdlg ret_size:
255558360b13Sdlg 	data->wgd_size = size;
255658360b13Sdlg 	return ret;
255758360b13Sdlg }
255858360b13Sdlg 
255958360b13Sdlg int
256058360b13Sdlg wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
256158360b13Sdlg {
256258360b13Sdlg 	struct ifreq	*ifr = (struct ifreq *) data;
256358360b13Sdlg 	struct wg_softc	*sc = ifp->if_softc;
256458360b13Sdlg 	int		 ret = 0;
256558360b13Sdlg 
256658360b13Sdlg 	switch (cmd) {
256758360b13Sdlg 	case SIOCSWG:
25682646c3b9Sclaudio 		NET_UNLOCK();
256958360b13Sdlg 		ret = wg_ioctl_set(sc, (struct wg_data_io *) data);
25702646c3b9Sclaudio 		NET_LOCK();
257158360b13Sdlg 		break;
257258360b13Sdlg 	case SIOCGWG:
25732646c3b9Sclaudio 		NET_UNLOCK();
257458360b13Sdlg 		ret = wg_ioctl_get(sc, (struct wg_data_io *) data);
25752646c3b9Sclaudio 		NET_LOCK();
257658360b13Sdlg 		break;
257758360b13Sdlg 	/* Interface IOCTLs */
257858360b13Sdlg 	case SIOCSIFADDR:
257958360b13Sdlg 		SET(ifp->if_flags, IFF_UP);
258058360b13Sdlg 		/* FALLTHROUGH */
258158360b13Sdlg 	case SIOCSIFFLAGS:
258258360b13Sdlg 		if (ISSET(ifp->if_flags, IFF_UP))
258358360b13Sdlg 			ret = wg_up(sc);
258458360b13Sdlg 		else
258558360b13Sdlg 			wg_down(sc);
258658360b13Sdlg 		break;
258758360b13Sdlg 	case SIOCSIFMTU:
258858360b13Sdlg 		/* Arbitrary limits */
258958360b13Sdlg 		if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > 9000)
259058360b13Sdlg 			ret = EINVAL;
259158360b13Sdlg 		else
259258360b13Sdlg 			ifp->if_mtu = ifr->ifr_mtu;
259358360b13Sdlg 		break;
259458360b13Sdlg 	case SIOCADDMULTI:
259558360b13Sdlg 	case SIOCDELMULTI:
259658360b13Sdlg 		break;
259758360b13Sdlg 	default:
259858360b13Sdlg 		ret = ENOTTY;
259958360b13Sdlg 	}
260058360b13Sdlg 
260158360b13Sdlg 	return ret;
260258360b13Sdlg }
260358360b13Sdlg 
260458360b13Sdlg int
260558360b13Sdlg wg_up(struct wg_softc *sc)
260658360b13Sdlg {
260758360b13Sdlg 	struct wg_peer	*peer;
260858360b13Sdlg 	int		 ret = 0;
260958360b13Sdlg 
261058360b13Sdlg 	NET_ASSERT_LOCKED();
261158360b13Sdlg 	/*
261258360b13Sdlg 	 * We use IFF_RUNNING as an exclusive access here. We also may want
261358360b13Sdlg 	 * an exclusive sc_lock as wg_bind may write to sc_udp_port. We also
261458360b13Sdlg 	 * want to drop NET_LOCK as we want to call socreate, sobind, etc. Once
261558360b13Sdlg 	 * solock is no longer === NET_LOCK, we may be able to avoid this.
261658360b13Sdlg 	 */
261758360b13Sdlg 	if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING)) {
261858360b13Sdlg 		SET(sc->sc_if.if_flags, IFF_RUNNING);
261958360b13Sdlg 		NET_UNLOCK();
262058360b13Sdlg 
262158360b13Sdlg 		rw_enter_write(&sc->sc_lock);
262258360b13Sdlg 		/*
262358360b13Sdlg 		 * If we successfully bind the socket, then enable the timers
262458360b13Sdlg 		 * for the peer. This will send all staged packets and a
262558360b13Sdlg 		 * keepalive if necessary.
262658360b13Sdlg 		 */
262758360b13Sdlg 		ret = wg_bind(sc, &sc->sc_udp_port, &sc->sc_udp_rtable);
262858360b13Sdlg 		if (ret == 0) {
262958360b13Sdlg 			TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) {
263058360b13Sdlg 				wg_timers_enable(&peer->p_timers);
263158360b13Sdlg 				wg_queue_out(sc, peer);
263258360b13Sdlg 			}
263358360b13Sdlg 		}
263458360b13Sdlg 		rw_exit_write(&sc->sc_lock);
263558360b13Sdlg 
263658360b13Sdlg 		NET_LOCK();
263758360b13Sdlg 		if (ret != 0)
263858360b13Sdlg 			CLR(sc->sc_if.if_flags, IFF_RUNNING);
263958360b13Sdlg 	}
264058360b13Sdlg 	return ret;
264158360b13Sdlg }
264258360b13Sdlg 
264358360b13Sdlg void
264458360b13Sdlg wg_down(struct wg_softc *sc)
264558360b13Sdlg {
264658360b13Sdlg 	struct wg_peer	*peer;
264758360b13Sdlg 
264858360b13Sdlg 	NET_ASSERT_LOCKED();
264958360b13Sdlg 	if (!ISSET(sc->sc_if.if_flags, IFF_RUNNING))
265058360b13Sdlg 		return;
265158360b13Sdlg 	CLR(sc->sc_if.if_flags, IFF_RUNNING);
265258360b13Sdlg 	NET_UNLOCK();
265358360b13Sdlg 
265458360b13Sdlg 	/*
265558360b13Sdlg 	 * We only need a read lock here, as we aren't writing to anything
265658360b13Sdlg 	 * that isn't granularly locked.
265758360b13Sdlg 	 */
265858360b13Sdlg 	rw_enter_read(&sc->sc_lock);
265958360b13Sdlg 	TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) {
266058360b13Sdlg 		mq_purge(&peer->p_stage_queue);
266158360b13Sdlg 		wg_timers_disable(&peer->p_timers);
266258360b13Sdlg 	}
266358360b13Sdlg 
266458360b13Sdlg 	taskq_barrier(wg_handshake_taskq);
266558360b13Sdlg 	TAILQ_FOREACH(peer, &sc->sc_peer_seq, p_seq_entry) {
266658360b13Sdlg 		noise_remote_clear(&peer->p_remote);
266758360b13Sdlg 		wg_timers_event_reset_handshake_last_sent(&peer->p_timers);
266858360b13Sdlg 	}
266958360b13Sdlg 
267058360b13Sdlg 	wg_unbind(sc);
267158360b13Sdlg 	rw_exit_read(&sc->sc_lock);
267258360b13Sdlg 	NET_LOCK();
267358360b13Sdlg }
267458360b13Sdlg 
267558360b13Sdlg int
267658360b13Sdlg wg_clone_create(struct if_clone *ifc, int unit)
267758360b13Sdlg {
267858360b13Sdlg 	struct ifnet		*ifp;
267958360b13Sdlg 	struct wg_softc		*sc;
268058360b13Sdlg 	struct noise_upcall	 local_upcall;
268158360b13Sdlg 
268258360b13Sdlg 	KERNEL_ASSERT_LOCKED();
268358360b13Sdlg 
268458360b13Sdlg 	if (wg_counter == 0) {
268558360b13Sdlg 		wg_handshake_taskq = taskq_create("wg_handshake",
268658360b13Sdlg 		    2, IPL_NET, TASKQ_MPSAFE);
268758360b13Sdlg 		wg_crypt_taskq = taskq_create("wg_crypt",
268858360b13Sdlg 		    ncpus, IPL_NET, TASKQ_MPSAFE);
268958360b13Sdlg 
269058360b13Sdlg 		if (wg_handshake_taskq == NULL || wg_crypt_taskq == NULL) {
269158360b13Sdlg 			if (wg_handshake_taskq != NULL)
269258360b13Sdlg 				taskq_destroy(wg_handshake_taskq);
269358360b13Sdlg 			if (wg_crypt_taskq != NULL)
269458360b13Sdlg 				taskq_destroy(wg_crypt_taskq);
269558360b13Sdlg 			wg_handshake_taskq = NULL;
269658360b13Sdlg 			wg_crypt_taskq = NULL;
269758360b13Sdlg 			return ENOTRECOVERABLE;
269858360b13Sdlg 		}
269958360b13Sdlg 	}
270058360b13Sdlg 	wg_counter++;
270158360b13Sdlg 
270258360b13Sdlg 	if ((sc = malloc(sizeof(*sc), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL)
270358360b13Sdlg 		goto ret_00;
270458360b13Sdlg 
270558360b13Sdlg 	local_upcall.u_arg = sc;
270658360b13Sdlg 	local_upcall.u_remote_get = wg_remote_get;
270758360b13Sdlg 	local_upcall.u_index_set = wg_index_set;
270858360b13Sdlg 	local_upcall.u_index_drop = wg_index_drop;
270958360b13Sdlg 
271058360b13Sdlg 	TAILQ_INIT(&sc->sc_peer_seq);
271158360b13Sdlg 
271258360b13Sdlg 	/* sc_if is initialised after everything else */
271358360b13Sdlg 	arc4random_buf(&sc->sc_secret, sizeof(sc->sc_secret));
271458360b13Sdlg 
271558360b13Sdlg 	rw_init(&sc->sc_lock, "wg");
271658360b13Sdlg 	noise_local_init(&sc->sc_local, &local_upcall);
271758360b13Sdlg 	if (cookie_checker_init(&sc->sc_cookie, &wg_ratelimit_pool) != 0)
271858360b13Sdlg 		goto ret_01;
271958360b13Sdlg 	sc->sc_udp_port = 0;
272058360b13Sdlg 	sc->sc_udp_rtable = 0;
272158360b13Sdlg 
272258360b13Sdlg 	rw_init(&sc->sc_so_lock, "wg_so");
272358360b13Sdlg 	sc->sc_so4 = NULL;
272458360b13Sdlg #ifdef INET6
272558360b13Sdlg 	sc->sc_so6 = NULL;
272658360b13Sdlg #endif
272758360b13Sdlg 
272858360b13Sdlg 	sc->sc_aip_num = 0;
272958360b13Sdlg 	if ((sc->sc_aip4 = art_alloc(0, 32, 0)) == NULL)
273058360b13Sdlg 		goto ret_02;
273158360b13Sdlg #ifdef INET6
273258360b13Sdlg 	if ((sc->sc_aip6 = art_alloc(0, 128, 0)) == NULL)
273358360b13Sdlg 		goto ret_03;
273458360b13Sdlg #endif
273558360b13Sdlg 
273658360b13Sdlg 	rw_init(&sc->sc_peer_lock, "wg_peer");
273758360b13Sdlg 	sc->sc_peer_num = 0;
273858360b13Sdlg 	if ((sc->sc_peer = hashinit(HASHTABLE_PEER_SIZE, M_DEVBUF,
273958360b13Sdlg 	    M_NOWAIT, &sc->sc_peer_mask)) == NULL)
274058360b13Sdlg 		goto ret_04;
274158360b13Sdlg 
274258360b13Sdlg 	mtx_init(&sc->sc_index_mtx, IPL_NET);
274358360b13Sdlg 	if ((sc->sc_index = hashinit(HASHTABLE_INDEX_SIZE, M_DEVBUF,
274458360b13Sdlg 	    M_NOWAIT, &sc->sc_index_mask)) == NULL)
274558360b13Sdlg 		goto ret_05;
274658360b13Sdlg 
274758360b13Sdlg 	task_set(&sc->sc_handshake, wg_handshake_worker, sc);
274858360b13Sdlg 	mq_init(&sc->sc_handshake_queue, MAX_QUEUED_HANDSHAKES, IPL_NET);
274958360b13Sdlg 
275058360b13Sdlg 	task_set(&sc->sc_encap, wg_encap_worker, sc);
275158360b13Sdlg 	task_set(&sc->sc_decap, wg_decap_worker, sc);
275258360b13Sdlg 
275358360b13Sdlg 	bzero(&sc->sc_encap_ring, sizeof(sc->sc_encap_ring));
275458360b13Sdlg 	mtx_init(&sc->sc_encap_ring.r_mtx, IPL_NET);
275558360b13Sdlg 	bzero(&sc->sc_decap_ring, sizeof(sc->sc_decap_ring));
275658360b13Sdlg 	mtx_init(&sc->sc_decap_ring.r_mtx, IPL_NET);
275758360b13Sdlg 
275858360b13Sdlg 	/* We've setup the softc, now we can setup the ifnet */
275958360b13Sdlg 	ifp = &sc->sc_if;
276058360b13Sdlg 	ifp->if_softc = sc;
276158360b13Sdlg 
276258360b13Sdlg 	snprintf(ifp->if_xname, sizeof(ifp->if_xname), "wg%d", unit);
276358360b13Sdlg 
276458360b13Sdlg 	ifp->if_mtu = DEFAULT_MTU;
276558360b13Sdlg 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_NOARP;
27668569c6f6Stobhe 	ifp->if_xflags = IFXF_CLONED | IFXF_MPSAFE;
27673e93ec8dStobhe 	ifp->if_txmit = 64; /* Keep our workers active for longer. */
276858360b13Sdlg 
276958360b13Sdlg 	ifp->if_ioctl = wg_ioctl;
27708569c6f6Stobhe 	ifp->if_qstart = wg_qstart;
277158360b13Sdlg 	ifp->if_output = wg_output;
277258360b13Sdlg 
277358360b13Sdlg 	ifp->if_type = IFT_WIREGUARD;
277433947fcbSkn 	ifp->if_rtrequest = p2p_rtrequest;
277558360b13Sdlg 
2776b5b8bc53Smvs 	if_counters_alloc(ifp);
277758360b13Sdlg 	if_attach(ifp);
277858360b13Sdlg 	if_alloc_sadl(ifp);
277958360b13Sdlg 
278058360b13Sdlg #if NBPFILTER > 0
278158360b13Sdlg 	bpfattach(&ifp->if_bpf, ifp, DLT_LOOP, sizeof(uint32_t));
278258360b13Sdlg #endif
278358360b13Sdlg 
2784*f08653c5Smvs 	WGPRINTF(LOG_INFO, sc, NULL, "Interface created\n");
278558360b13Sdlg 
278658360b13Sdlg 	return 0;
278758360b13Sdlg ret_05:
278858360b13Sdlg 	hashfree(sc->sc_peer, HASHTABLE_PEER_SIZE, M_DEVBUF);
278958360b13Sdlg ret_04:
279058360b13Sdlg #ifdef INET6
279158360b13Sdlg 	free(sc->sc_aip6, M_RTABLE, sizeof(*sc->sc_aip6));
279258360b13Sdlg ret_03:
279358360b13Sdlg #endif
279458360b13Sdlg 	free(sc->sc_aip4, M_RTABLE, sizeof(*sc->sc_aip4));
279558360b13Sdlg ret_02:
279658360b13Sdlg 	cookie_checker_deinit(&sc->sc_cookie);
279758360b13Sdlg ret_01:
279858360b13Sdlg 	free(sc, M_DEVBUF, sizeof(*sc));
279958360b13Sdlg ret_00:
280058360b13Sdlg 	return ENOBUFS;
280158360b13Sdlg }
280258360b13Sdlg int
280358360b13Sdlg wg_clone_destroy(struct ifnet *ifp)
280458360b13Sdlg {
280558360b13Sdlg 	struct wg_softc	*sc = ifp->if_softc;
280658360b13Sdlg 	struct wg_peer	*peer, *tpeer;
280758360b13Sdlg 
280858360b13Sdlg 	KERNEL_ASSERT_LOCKED();
280958360b13Sdlg 
281058360b13Sdlg 	rw_enter_write(&sc->sc_lock);
281158360b13Sdlg 	TAILQ_FOREACH_SAFE(peer, &sc->sc_peer_seq, p_seq_entry, tpeer)
281258360b13Sdlg 		wg_peer_destroy(peer);
281358360b13Sdlg 	rw_exit_write(&sc->sc_lock);
281458360b13Sdlg 
281558360b13Sdlg 	wg_unbind(sc);
281658360b13Sdlg 	if_detach(ifp);
281758360b13Sdlg 
281858360b13Sdlg 	wg_counter--;
281958360b13Sdlg 	if (wg_counter == 0) {
282058360b13Sdlg 		KASSERT(wg_handshake_taskq != NULL && wg_crypt_taskq != NULL);
282158360b13Sdlg 		taskq_destroy(wg_handshake_taskq);
282258360b13Sdlg 		taskq_destroy(wg_crypt_taskq);
282358360b13Sdlg 		wg_handshake_taskq = NULL;
282458360b13Sdlg 		wg_crypt_taskq = NULL;
282558360b13Sdlg 	}
282658360b13Sdlg 
2827*f08653c5Smvs 	WGPRINTF(LOG_INFO, sc, NULL, "Interface destroyed\n");
282858360b13Sdlg 
282958360b13Sdlg 	hashfree(sc->sc_index, HASHTABLE_INDEX_SIZE, M_DEVBUF);
283058360b13Sdlg 	hashfree(sc->sc_peer, HASHTABLE_PEER_SIZE, M_DEVBUF);
283158360b13Sdlg #ifdef INET6
283258360b13Sdlg 	free(sc->sc_aip6, M_RTABLE, sizeof(*sc->sc_aip6));
283358360b13Sdlg #endif
283458360b13Sdlg 	free(sc->sc_aip4, M_RTABLE, sizeof(*sc->sc_aip4));
283558360b13Sdlg 	cookie_checker_deinit(&sc->sc_cookie);
283658360b13Sdlg 	free(sc, M_DEVBUF, sizeof(*sc));
283758360b13Sdlg 	return 0;
283858360b13Sdlg }
283958360b13Sdlg 
284058360b13Sdlg void
284158360b13Sdlg wgattach(int nwg)
284258360b13Sdlg {
284358360b13Sdlg #ifdef WGTEST
284458360b13Sdlg 	cookie_test();
284558360b13Sdlg 	noise_test();
284658360b13Sdlg #endif
284758360b13Sdlg 	if_clone_attach(&wg_cloner);
284858360b13Sdlg 
284958360b13Sdlg 	pool_init(&wg_aip_pool, sizeof(struct wg_aip), 0,
285058360b13Sdlg 			IPL_NET, 0, "wgaip", NULL);
285158360b13Sdlg 	pool_init(&wg_peer_pool, sizeof(struct wg_peer), 0,
285258360b13Sdlg 			IPL_NET, 0, "wgpeer", NULL);
285358360b13Sdlg 	pool_init(&wg_ratelimit_pool, sizeof(struct ratelimit_entry), 0,
285458360b13Sdlg 			IPL_NET, 0, "wgratelimit", NULL);
285558360b13Sdlg }
2856