350 	struct wg_packet *pkt;  in wg_packet_alloc()  local
352 	if ((pkt = objcache_get(wg_packet_zone, M_NOWAIT)) == NULL)  in wg_packet_alloc()
355 	bzero(pkt, sizeof(*pkt)); /* objcache_get() doesn't ensure M_ZERO. */  in wg_packet_alloc()
356 	pkt->p_mbuf = m;  in wg_packet_alloc()
358 	return (pkt);  in wg_packet_alloc()
362 wg_packet_free(struct wg_packet *pkt)  in wg_packet_free()  argument
364 	if (pkt->p_keypair != NULL)  in wg_packet_free()
365 		noise_keypair_put(pkt->p_keypair);  in wg_packet_free()
366 	if (pkt->p_mbuf != NULL)  in wg_packet_free()
367 		m_freem(pkt->p_mbuf);  in wg_packet_free()
368 	objcache_put(wg_packet_zone, pkt);  in wg_packet_free()
420 wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt)  in wg_queue_enqueue_handshake()  argument
426 		STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel);  in wg_queue_enqueue_handshake()
433 		wg_packet_free(pkt);  in wg_queue_enqueue_handshake()
441 	struct wg_packet *pkt;  in wg_queue_dequeue_handshake()  local
444 	pkt = STAILQ_FIRST(&hs->q_queue);  in wg_queue_dequeue_handshake()
445 	if (pkt != NULL) {  in wg_queue_dequeue_handshake()
451 	return (pkt);  in wg_queue_dequeue_handshake()
455 wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt)  in wg_queue_push_staged()  argument
465 	STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel);  in wg_queue_push_staged()
476 	struct wg_packet *pkt, *tpkt;  in wg_queue_enlist_staged()  local
478 	STAILQ_FOREACH_MUTABLE(pkt, list, p_parallel, tpkt)  in wg_queue_enlist_staged()
479 		wg_queue_push_staged(staged, pkt);  in wg_queue_enlist_staged()
497 	struct wg_packet *pkt, *tpkt;  in wg_queue_purge()  local
500 	STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt)  in wg_queue_purge()
501 		wg_packet_free(pkt);  in wg_queue_purge()
506 	      struct wg_packet *pkt)  in wg_queue_both()  argument
508 	pkt->p_state = WG_PACKET_UNCRYPTED;  in wg_queue_both()
513 		STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial);  in wg_queue_both()
516 		wg_packet_free(pkt);  in wg_queue_both()
524 		STAILQ_INSERT_TAIL(&parallel->q_queue, pkt, p_parallel);  in wg_queue_both()
532 		pkt->p_state = WG_PACKET_DEAD;  in wg_queue_both()
543 	struct wg_packet *pkt = NULL;  in wg_queue_dequeue_serial()  local
553 		pkt = STAILQ_FIRST(&serial->q_queue);  in wg_queue_dequeue_serial()
558 	return (pkt);  in wg_queue_dequeue_serial()
564 	struct wg_packet *pkt = NULL;  in wg_queue_dequeue_parallel()  local
569 		pkt = STAILQ_FIRST(&parallel->q_queue);  in wg_queue_dequeue_parallel()
574 	return (pkt);  in wg_queue_dequeue_parallel()
1489 	struct wg_pkt_initiation pkt;  in wg_send_initiation()  local
1491 	if (!noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue,  in wg_send_initiation()
1492 				     pkt.es, pkt.ets))  in wg_send_initiation()
1498 	pkt.t = WG_PKT_INITIATION;  in wg_send_initiation()
1499 	cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt,  in wg_send_initiation()
1500 			 sizeof(pkt) - sizeof(pkt.m));  in wg_send_initiation()
1501 	wg_peer_send_buf(peer, &pkt, sizeof(pkt));  in wg_send_initiation()
1508 	struct wg_pkt_response pkt;  in wg_send_response()  local
1510 	if (!noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx,  in wg_send_response()
1511 				   pkt.ue, pkt.en))  in wg_send_response()
1518 	pkt.t = WG_PKT_RESPONSE;  in wg_send_response()
1519 	cookie_maker_mac(peer->p_cookie, &pkt.m, &pkt,  in wg_send_response()
1520 			 sizeof(pkt) - sizeof(pkt.m));  in wg_send_response()
1521 	wg_peer_send_buf(peer, &pkt, sizeof(pkt));  in wg_send_response()
1528 	struct wg_pkt_cookie pkt;  in wg_send_cookie()  local
1532 	pkt.t = WG_PKT_COOKIE;  in wg_send_cookie()
1533 	pkt.r_idx = idx;  in wg_send_cookie()
1535 	cookie_checker_create_payload(sc->sc_cookie, cm, pkt.nonce,  in wg_send_cookie()
1536 				      pkt.ec, &e->e_remote.r_sa);  in wg_send_cookie()
1537 	wg_send_buf(sc, e, &pkt, sizeof(pkt));  in wg_send_cookie()
1543 	struct wg_packet *pkt;  in wg_send_keepalive()  local
1550 	if ((pkt = wg_packet_alloc(m)) == NULL) {  in wg_send_keepalive()
1555 	wg_queue_push_staged(&peer->p_stage_queue, pkt);  in wg_send_keepalive()
1589 wg_handshake(struct wg_softc *sc, struct wg_packet *pkt)  in wg_handshake()  argument
1601 	pkt->p_mbuf = m_pullup(pkt->p_mbuf, pkt->p_mbuf->m_pkthdr.len);  in wg_handshake()
1602 	if (pkt->p_mbuf == NULL)  in wg_handshake()
1606 	m = pkt->p_mbuf;  in wg_handshake()
1607 	e = &pkt->p_endpoint;  in wg_handshake()
1737 	wg_packet_free(pkt);  in wg_handshake()
1745 	wg_packet_free(pkt);  in wg_handshake()
1753 	struct wg_packet	*pkt;  in wg_handshake_worker()  local
1755 	while ((pkt = wg_queue_dequeue_handshake(queue)) != NULL)  in wg_handshake_worker()
1756 		wg_handshake(sc, pkt);  in wg_handshake_worker()
1781 calculate_padding(struct wg_packet *pkt)  in calculate_padding()  argument
1785 	last_unit = pkt->p_mbuf->m_pkthdr.len;  in calculate_padding()
1788 	if (__predict_false(pkt->p_mtu == 0))  in calculate_padding()
1795 	if (__predict_false(last_unit > pkt->p_mtu))  in calculate_padding()
1796 		last_unit %= pkt->p_mtu;  in calculate_padding()
1798 	padded_size = MIN(pkt->p_mtu, WG_PKT_WITH_PADDING(last_unit));  in calculate_padding()
1831 wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt)  in wg_encrypt()  argument
1841 	remote = noise_keypair_remote(pkt->p_keypair);  in wg_encrypt()
1843 	m = pkt->p_mbuf;  in wg_encrypt()
1845 	padlen = calculate_padding(pkt);  in wg_encrypt()
1849 	if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_counter, m) != 0)  in wg_encrypt()
1858 	data->counter = htole64(pkt->p_counter);  in wg_encrypt()
1863 	pkt->p_mbuf = m;  in wg_encrypt()
1864 	atomic_store_rel_int(&pkt->p_state, state);  in wg_encrypt()
1870 wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt)  in wg_decrypt()  argument
1878 	remote = noise_keypair_remote(pkt->p_keypair);  in wg_decrypt()
1880 	m = pkt->p_mbuf;  in wg_decrypt()
1882 	pkt->p_counter = le64toh(mtod(m, struct wg_pkt_data *)->counter);  in wg_decrypt()
1885 	if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_counter, m) != 0)  in wg_decrypt()
1900 	if (determine_af_and_pullup(&m, &pkt->p_af) != 0)  in wg_decrypt()
1902 	if (pkt->p_af == AF_INET) {  in wg_decrypt()
1929 	pkt->p_mbuf = m;  in wg_decrypt()
1930 	atomic_store_rel_int(&pkt->p_state, state);  in wg_decrypt()
1940 	struct wg_packet	*pkt;  in wg_encrypt_worker()  local
1942 	while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL)  in wg_encrypt_worker()
1943 		wg_encrypt(sc, pkt);  in wg_encrypt_worker()
1951 	struct wg_packet	*pkt;  in wg_decrypt_worker()  local
1953 	while ((pkt = wg_queue_dequeue_parallel(queue)) != NULL)  in wg_decrypt_worker()
1954 		wg_decrypt(sc, pkt);  in wg_decrypt_worker()
1989 	struct wg_packet	*pkt;  in wg_deliver_out()  local
1995 	while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) {  in wg_deliver_out()
1996 		if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED) {  in wg_deliver_out()
1998 			wg_packet_free(pkt);  in wg_deliver_out()
2002 		m = pkt->p_mbuf;  in wg_deliver_out()
2006 		pkt->p_mbuf = NULL;  in wg_deliver_out()
2007 		wg_packet_free(pkt);  in wg_deliver_out()
2034 	struct wg_packet	*pkt;  in wg_deliver_in()  local
2043 	while ((pkt = wg_queue_dequeue_serial(queue)) != NULL) {  in wg_deliver_in()
2044 		if (atomic_load_acq_int(&pkt->p_state) != WG_PACKET_CRYPTED ||  in wg_deliver_in()
2045 		    noise_keypair_counter_check(pkt->p_keypair, pkt->p_counter)  in wg_deliver_in()
2048 			wg_packet_free(pkt);  in wg_deliver_in()
2052 		if (noise_keypair_received_with(pkt->p_keypair))  in wg_deliver_in()
2057 		wg_peer_set_endpoint(peer, &pkt->p_endpoint);  in wg_deliver_in()
2059 		m = pkt->p_mbuf;  in wg_deliver_in()
2085 			wg_bpf_ptap(ifp, m, pkt->p_af);  in wg_deliver_in()
2087 			netisr_queue((pkt->p_af == AF_INET ?  in wg_deliver_in()
2089 			pkt->p_mbuf = NULL;  in wg_deliver_in()
2094 		wg_packet_free(pkt);  in wg_deliver_in()
2106 	struct wg_packet	*pkt;  in wg_input()  local
2132 	if ((pkt = wg_packet_alloc(m)) == NULL) {  in wg_input()
2141 		pkt->p_endpoint.e_remote.r_sin =  in wg_input()
2146 		pkt->p_endpoint.e_remote.r_sin6 =  in wg_input()
2158 		if (!wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt)) {  in wg_input()
2169 		pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data));  in wg_input()
2170 		if (pkt->p_mbuf == NULL)  in wg_input()
2173 		data = mtod(pkt->p_mbuf, struct wg_pkt_data *);  in wg_input()
2174 		pkt->p_keypair = noise_keypair_lookup(sc->sc_local,  in wg_input()
2176 		if (pkt->p_keypair == NULL)  in wg_input()
2179 		remote = noise_keypair_remote(pkt->p_keypair);  in wg_input()
2182 				   &peer->p_decrypt_serial, pkt))  in wg_input()
2192 	wg_packet_free(pkt);  in wg_input()
2225 	struct wg_packet	*pkt, *tpkt;  in wg_peer_send_staged()  local
2242 	STAILQ_FOREACH(pkt, &list, p_parallel) {  in wg_peer_send_staged()
2243 		if (!noise_keypair_counter_next(keypair, &pkt->p_counter))  in wg_peer_send_staged()
2246 	STAILQ_FOREACH_MUTABLE(pkt, &list, p_parallel, tpkt) {  in wg_peer_send_staged()
2247 		pkt->p_keypair = noise_keypair_ref(keypair);  in wg_peer_send_staged()
2249 				   &peer->p_encrypt_serial, pkt))  in wg_peer_send_staged()
2269 	struct wg_packet	*pkt = NULL;  in wg_output()  local
2315 	if ((pkt = wg_packet_alloc(m)) == NULL) {  in wg_output()
2320 	pkt->p_af = af;  in wg_output()
2321 	pkt->p_mtu = ifp->if_mtu;  in wg_output()
2323 	    rt->rt_rmx.rmx_mtu < pkt->p_mtu)  in wg_output()
2324 		pkt->p_mtu = rt->rt_rmx.rmx_mtu;  in wg_output()
2342 	wg_queue_push_staged(&peer->p_stage_queue, pkt);  in wg_output()
2365 	if (pkt != NULL) {  in wg_output()
2366 		pkt->p_mbuf = NULL; /* m already freed above */  in wg_output()
2367 		wg_packet_free(pkt);  in wg_output()