Lines Matching defs:so
346 * Now do a deep copy of the variable-length arrays in the struct, so that
349 * error paths so that our caller need only worry about outstanding
405 ktls_get_cpu(struct socket *so)
413 inp = sotoinpcb(so);
599 ktls_create_session(struct socket *so, struct tls_enable *en,
710 tls->inp = so->so_pcb;
715 tls->wq_index = ktls_get_cpu(so);
815 * array so that it is 8-byte aligned.
869 ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)
875 inp = so->so_pcb;
1096 ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction,
1104 error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
1110 error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst);
1286 ktls_enable_rx(struct socket *so, struct tls_enable *en)
1300 if (so->so_proto->pr_protocol != IPPROTO_TCP)
1307 if (so->so_rcv.sb_tls_info != NULL)
1313 error = ktls_create_session(so, en, &tls, KTLS_RX);
1327 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT);
1334 SOCK_RECVBUF_LOCK(so);
1335 if (__predict_false(so->so_rcv.sb_tls_info != NULL)) {
1336 SOCK_RECVBUF_UNLOCK(so);
1337 SOCK_IO_RECV_UNLOCK(so);
1341 so->so_rcv.sb_tls_seqno = be64dec(en->rec_seq);
1342 so->so_rcv.sb_tls_info = tls;
1343 so->so_rcv.sb_flags |= SB_TLS_RX;
1346 sb_mark_notready(&so->so_rcv);
1347 ktls_check_rx(&so->so_rcv);
1348 SOCK_RECVBUF_UNLOCK(so);
1349 SOCK_IO_RECV_UNLOCK(so);
1353 error = ktls_try_toe(so, tls, KTLS_RX);
1356 error = ktls_try_ifnet(so, tls, KTLS_RX, false);
1366 ktls_enable_tx(struct socket *so, struct tls_enable *en)
1382 if (so->so_proto->pr_protocol != IPPROTO_TCP)
1389 if (so->so_snd.sb_tls_info != NULL)
1399 error = ktls_create_session(so, en, &tls, KTLS_TX);
1405 error = ktls_try_toe(so, tls, KTLS_TX);
1408 error = ktls_try_ifnet(so, tls, KTLS_TX, false);
1421 error = SOCK_IO_SEND_LOCK(so, SBL_WAIT);
1428 * Write lock the INP when setting sb_tls_info so that
1432 inp = so->so_pcb;
1434 SOCK_SENDBUF_LOCK(so);
1435 if (__predict_false(so->so_snd.sb_tls_info != NULL)) {
1436 SOCK_SENDBUF_UNLOCK(so);
1438 SOCK_IO_SEND_UNLOCK(so);
1442 so->so_snd.sb_tls_seqno = be64dec(en->rec_seq);
1443 so->so_snd.sb_tls_info = tls;
1451 SOCK_SENDBUF_UNLOCK(so);
1453 SOCK_IO_SEND_UNLOCK(so);
1461 ktls_get_rx_mode(struct socket *so, int *modep)
1466 if (SOLISTENING(so))
1468 inp = so->so_pcb;
1470 SOCK_RECVBUF_LOCK(so);
1471 tls = so->so_rcv.sb_tls_info;
1476 SOCK_RECVBUF_UNLOCK(so);
1496 struct socket *so;
1500 so = inp->inp_socket;
1501 if (__predict_false(so == NULL)) {
1513 SOCKBUF_LOCK(&so->so_rcv);
1514 *tcpseq = tp->rcv_nxt - so->so_rcv.sb_tlscc;
1515 *tlsseq = so->so_rcv.sb_tls_seqno;
1516 SOCKBUF_UNLOCK(&so->so_rcv);
1524 ktls_get_tx_mode(struct socket *so, int *modep)
1529 if (SOLISTENING(so))
1531 inp = so->so_pcb;
1533 SOCK_SENDBUF_LOCK(so);
1534 tls = so->so_snd.sb_tls_info;
1539 SOCK_SENDBUF_UNLOCK(so);
1547 ktls_set_tx_mode(struct socket *so, int mode)
1554 if (SOLISTENING(so))
1564 inp = so->so_pcb;
1581 SOCKBUF_LOCK(&so->so_snd);
1582 tls = so->so_snd.sb_tls_info;
1584 SOCKBUF_UNLOCK(&so->so_snd);
1589 SOCKBUF_UNLOCK(&so->so_snd);
1594 SOCKBUF_UNLOCK(&so->so_snd);
1600 error = ktls_try_ifnet(so, tls_new, KTLS_TX, true);
1611 error = SOCK_IO_SEND_LOCK(so, SBL_WAIT);
1624 if (tls != so->so_snd.sb_tls_info) {
1626 SOCK_IO_SEND_UNLOCK(so);
1634 SOCKBUF_LOCK(&so->so_snd);
1635 so->so_snd.sb_tls_info = tls_new;
1642 SOCKBUF_UNLOCK(&so->so_snd);
1643 SOCK_IO_SEND_UNLOCK(so);
1676 struct socket *so;
1682 so = tls->so;
1683 inp = so->so_pcb;
1692 SOCKBUF_LOCK(&so->so_rcv);
1700 SOCKBUF_UNLOCK(&so->so_rcv);
1721 SOCKBUF_LOCK(&so->so_rcv);
1723 SOCKBUF_UNLOCK(&so->so_rcv);
1743 CURVNET_SET(so->so_vnet);
1744 sorele(so);
1836 struct socket *so;
1841 so = __containerof(sb, struct socket, so_rcv);
1855 soref(so);
1856 tls->so = so;
1952 * We might hold the rlock, so let's
2221 struct socket *so;
2227 so = __containerof(sb, struct socket, so_rcv);
2235 so->so_error = EMSGSIZE;
2244 so->so_error = EMSGSIZE;
2250 soref(so);
2251 wq = &ktls_wq[so->so_rcv.sb_tls_info->wq_index];
2253 STAILQ_INSERT_TAIL(&wq->so_head, so, so_ktls_rx_list);
2284 * accounting, so do it inline instead.
2362 * Walking the mbuf chain backwards is clumsy, so another option would
2447 ktls_resync_ifnet(struct socket *so, uint32_t tls_len, uint64_t tls_rcd_num)
2454 mst = so->so_rcv.sb_tls_info->snd_tag;
2458 inp = sotoinpcb(so);
2472 SOCKBUF_LOCK(&so->so_rcv);
2474 tp->rcv_nxt - so->so_rcv.sb_tlscc - tls_len;
2477 SOCKBUF_UNLOCK(&so->so_rcv);
2486 ktls_drop(struct socket *so, int error)
2489 struct inpcb *inp = sotoinpcb(so);
2502 so->so_error = error;
2503 SOCK_RECVBUF_LOCK(so);
2504 sorwakeup_locked(so);
2511 ktls_decrypt(struct socket *so)
2526 sb = &so->so_rcv;
2529 ("%s: socket %p not running", __func__, so));
2562 * recoverable at this point, so abort it.
2567 ktls_drop(so, error);
2652 CURVNET_SET(so->so_vnet);
2653 so->so_error = error;
2654 sorwakeup_locked(so);
2728 ktls_resync_ifnet(so, tls_len, seqno);
2733 ktls_resync_ifnet(so, 0, seqno);
2741 so->so_error = EMSGSIZE;
2743 sorwakeup_locked(so);
2748 CURVNET_SET(so->so_vnet);
2749 sorele(so);
2890 ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
2910 m->m_epg_so = so;
2929 * it must be a future record, so insert it, sorted by
3027 struct socket *so;
3031 so = top->m_epg_so;
3034 KASSERT(so != NULL, ("so = NULL, top = %p\n", top));
3090 CURVNET_SET(so->so_vnet);
3092 (void)so->so_proto->pr_ready(so, top, npages);
3094 ktls_drop(so, EIO);
3098 sorele(so);
3106 struct socket *so;
3116 so = state->so;
3132 CURVNET_SET(so->so_vnet);
3136 (void)so->so_proto->pr_ready(so, m, npages);
3138 ktls_drop(so, EIO);
3142 sorele(so);
3156 struct socket *so;
3160 so = top->m_epg_so;
3163 KASSERT(so != NULL, ("so = NULL, top = %p\n", top));
3180 soref(so);
3181 state->so = so;
3191 CURVNET_SET(so->so_vnet);
3192 sorele(so);
3200 CURVNET_SET(so->so_vnet);
3202 ktls_drop(so, EIO);
3206 sorele(so);
3278 struct socket *so, *son;
3339 STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) {
3340 ktls_decrypt(so);
3352 struct socket *so;
3360 so = inp->inp_socket;
3361 MPASS(so != NULL);
3366 if (so->so_snd.sb_tls_info != NULL)
3367 err = ktls_set_tx_mode(so, TCP_TLS_MODE_SW);
3372 /* ktls_set_tx_mode() drops inp wlock, so recheck flags */
3382 CURVNET_SET(so->so_vnet);
3383 sorele(so);
3407 struct socket *so;
3413 so = inp->inp_socket;
3414 SOCK_LOCK(so);
3415 tls = so->so_snd.sb_tls_info;
3417 SOCK_UNLOCK(so);
3423 * ifnet can only be done once per connection, so we never want
3428 soref(so);
3430 SOCK_UNLOCK(so);