Lines Matching defs:tls
111 SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
306 ktls_copyin_tls_enable(struct sockopt *sopt, struct tls_enable *tls)
316 memset(tls, 0, sizeof(*tls));
317 tls->cipher_key = tls_v0.cipher_key;
318 tls->iv = tls_v0.iv;
319 tls->auth_key = tls_v0.auth_key;
320 tls->cipher_algorithm = tls_v0.cipher_algorithm;
321 tls->cipher_key_len = tls_v0.cipher_key_len;
322 tls->iv_len = tls_v0.iv_len;
323 tls->auth_algorithm = tls_v0.auth_algorithm;
324 tls->auth_key_len = tls_v0.auth_key_len;
325 tls->flags = tls_v0.flags;
326 tls->tls_vmajor = tls_v0.tls_vmajor;
327 tls->tls_vminor = tls_v0.tls_vminor;
329 error = sooptcopyin(sopt, tls, sizeof(*tls), sizeof(*tls));
334 if (tls->cipher_key_len < 0 || tls->cipher_key_len > TLS_MAX_PARAM_SIZE)
336 if (tls->iv_len < 0 || tls->iv_len > sizeof(((struct ktls_session *)NULL)->params.iv))
338 if (tls->auth_key_len < 0 || tls->auth_key_len > TLS_MAX_PARAM_SIZE)
342 if (tls->cipher_key_len == 0)
352 if (tls->cipher_key_len != 0) {
353 cipher_key = malloc(tls->cipher_key_len, M_KTLS, M_WAITOK);
355 error = copyin(tls->cipher_key, cipher_key, tls->cipher_key_len);
359 bcopy(tls->cipher_key, cipher_key, tls->cipher_key_len);
362 if (tls->iv_len != 0) {
363 iv = malloc(tls->iv_len, M_KTLS, M_WAITOK);
365 error = copyin(tls->iv, iv, tls->iv_len);
369 bcopy(tls->iv, iv, tls->iv_len);
372 if (tls->auth_key_len != 0) {
373 auth_key = malloc(tls->auth_key_len, M_KTLS, M_WAITOK);
375 error = copyin(tls->auth_key, auth_key, tls->auth_key_len);
379 bcopy(tls->auth_key, auth_key, tls->auth_key_len);
382 tls->cipher_key = cipher_key;
383 tls->iv = iv;
384 tls->auth_key = auth_key;
397 ktls_cleanup_tls_enable(struct tls_enable *tls)
399 zfree(__DECONST(void *, tls->cipher_key), M_KTLS);
400 zfree(__DECONST(void *, tls->iv), M_KTLS);
401 zfree(__DECONST(void *, tls->auth_key), M_KTLS);
602 struct ktls_session *tls;
701 tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO);
705 refcount_init(&tls->refcount, 1);
707 TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_receive_tag, tls);
709 TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);
710 tls->inp = so->so_pcb;
711 in_pcbref(tls->inp);
712 tls->tx = true;
715 tls->wq_index = ktls_get_cpu(so);
717 tls->params.cipher_algorithm = en->cipher_algorithm;
718 tls->params.auth_algorithm = en->auth_algorithm;
719 tls->params.tls_vmajor = en->tls_vmajor;
720 tls->params.tls_vminor = en->tls_vminor;
721 tls->params.flags = en->flags;
722 tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2, ktls_maxlen);
725 tls->params.tls_hlen = sizeof(struct tls_record_layer);
733 tls->params.tls_hlen += sizeof(uint64_t);
734 tls->params.tls_tlen = AES_GMAC_HASH_LEN;
735 tls->params.tls_bs = 1;
742 tls->sequential_records = true;
743 tls->next_seqno = be64dec(en->rec_seq);
744 STAILQ_INIT(&tls->pending_records);
746 tls->params.tls_hlen += AES_BLOCK_LEN;
748 tls->params.tls_tlen = AES_BLOCK_LEN +
752 tls->params.tls_hlen += AES_BLOCK_LEN;
753 tls->params.tls_tlen = AES_BLOCK_LEN +
757 tls->params.tls_hlen += AES_BLOCK_LEN;
758 tls->params.tls_tlen = AES_BLOCK_LEN +
764 tls->params.tls_bs = AES_BLOCK_LEN;
770 tls->params.tls_tlen = POLY1305_HASH_LEN;
771 tls->params.tls_bs = 1;
783 tls->params.tls_tlen += sizeof(uint8_t);
785 KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN,
786 ("TLS header length too long: %d", tls->params.tls_hlen));
787 KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN,
788 ("TLS trailer length too long: %d", tls->params.tls_tlen));
791 tls->params.auth_key_len = en->auth_key_len;
792 tls->params.auth_key = malloc(en->auth_key_len, M_KTLS,
794 bcopy(en->auth_key, tls->params.auth_key, en->auth_key_len);
797 tls->params.cipher_key_len = en->cipher_key_len;
798 tls->params.cipher_key = malloc(en->cipher_key_len, M_KTLS, M_WAITOK);
799 bcopy(en->cipher_key, tls->params.cipher_key, en->cipher_key_len);
807 tls->params.iv_len = en->iv_len;
808 bcopy(en->iv, tls->params.iv, en->iv_len);
819 arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0);
822 *tlsp = tls;
827 ktls_clone_session(struct ktls_session *tls, int direction)
842 tls_new->inp = tls->inp;
848 tls_new->params = tls->params;
849 tls_new->wq_index = tls->wq_index;
853 tls_new->params.auth_key = malloc(tls->params.auth_key_len,
855 memcpy(tls_new->params.auth_key, tls->params.auth_key,
856 tls->params.auth_key_len);
859 tls_new->params.cipher_key = malloc(tls->params.cipher_key_len, M_KTLS,
861 memcpy(tls_new->params.cipher_key, tls->params.cipher_key,
862 tls->params.cipher_key_len);
869 ktls_try_toe(struct socket *so, struct ktls_session *tls, int direction)
891 error = tcp_offload_alloc_tls_session(tp, tls, direction);
894 tls->mode = TCP_TLS_MODE_TOE;
895 switch (tls->params.cipher_algorithm) {
918 ktls_alloc_snd_tag(struct inpcb *inp, struct ktls_session *tls, bool force,
973 params.tls_rate_limit.tls = tls;
977 params.tls.inp = inp;
978 params.tls.tls = tls;
1016 ktls_alloc_rcv_tag(struct inpcb *inp, struct ktls_session *tls,
1024 if (!ktls_ocf_recrypt_supported(tls))
1057 tls->rx_ifp = ifp;
1064 params.tls_rx.tls = tls;
1090 tls->rx_vlan_id = params.tls_rx.vlan_id;
1096 ktls_try_ifnet(struct socket *so, struct ktls_session *tls, int direction,
1104 error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
1110 error = ktls_alloc_rcv_tag(so->so_pcb, tls, &mst);
1118 tls->mode = TCP_TLS_MODE_IFNET;
1119 tls->snd_tag = mst;
1121 switch (tls->params.cipher_algorithm) {
1139 ktls_use_sw(struct ktls_session *tls)
1141 tls->mode = TCP_TLS_MODE_SW;
1142 switch (tls->params.cipher_algorithm) {
1156 ktls_try_sw(struct ktls_session *tls, int direction)
1160 error = ktls_ocf_try(tls, direction);
1163 ktls_use_sw(tls);
1288 struct ktls_session *tls;
1313 error = ktls_create_session(so, en, &tls, KTLS_RX);
1317 error = ktls_ocf_try(tls, KTLS_RX);
1319 ktls_free(tls);
1329 ktls_free(tls);
1338 ktls_free(tls);
1342 so->so_rcv.sb_tls_info = tls;
1353 error = ktls_try_toe(so, tls, KTLS_RX);
1356 error = ktls_try_ifnet(so, tls, KTLS_RX, false);
1358 ktls_use_sw(tls);
1368 struct ktls_session *tls;
1399 error = ktls_create_session(so, en, &tls, KTLS_TX);
1405 error = ktls_try_toe(so, tls, KTLS_TX);
1408 error = ktls_try_ifnet(so, tls, KTLS_TX, false);
1410 error = ktls_try_sw(tls, KTLS_TX);
1413 ktls_free(tls);
1423 ktls_free(tls);
1439 ktls_free(tls);
1443 so->so_snd.sb_tls_info = tls;
1444 if (tls->mode != TCP_TLS_MODE_SW) {
1463 struct ktls_session *tls;
1471 tls = so->so_rcv.sb_tls_info;
1472 if (tls == NULL)
1475 *modep = tls->mode;
1526 struct ktls_session *tls;
1534 tls = so->so_snd.sb_tls_info;
1535 if (tls == NULL)
1538 *modep = tls->mode;
1549 struct ktls_session *tls, *tls_new;
1582 tls = so->so_snd.sb_tls_info;
1583 if (tls == NULL) {
1588 if (tls->mode == mode) {
1593 tls = ktls_hold(tls);
1597 tls_new = ktls_clone_session(tls, KTLS_TX);
1606 ktls_free(tls);
1615 ktls_free(tls);
1624 if (tls != so->so_snd.sb_tls_info) {
1628 ktls_free(tls);
1646 * Drop two references on 'tls'. The first is for the
1650 KASSERT(tls->refcount >= 2, ("too few references on old session"));
1651 ktls_free(tls);
1652 ktls_free(tls);
1672 struct ktls_session *tls;
1681 tls = context;
1682 so = tls->so;
1693 mst = tls->snd_tag;
1694 tls->snd_tag = NULL;
1698 ifp = tls->rx_ifp;
1707 params.tls_rx.tls = tls;
1708 params.tls_rx.vlan_id = tls->rx_vlan_id;
1722 tls->snd_tag = mst;
1737 mtx_pool_lock(mtxpool_sleep, tls);
1738 tls->reset_pending = false;
1739 mtx_pool_unlock(mtxpool_sleep, tls);
1746 ktls_free(tls);
1761 struct ktls_session *tls;
1769 tls = context;
1770 inp = tls->inp;
1778 * Write-lock the INP when changing tls->snd_tag since
1783 old = tls->snd_tag;
1784 tls->snd_tag = NULL;
1789 error = ktls_alloc_snd_tag(inp, tls, true, &new);
1793 tls->snd_tag = new;
1794 mtx_pool_lock(mtxpool_sleep, tls);
1795 tls->reset_pending = false;
1796 mtx_pool_unlock(mtxpool_sleep, tls);
1829 ktls_free(tls);
1835 struct ktls_session *tls;
1843 tls = sb->sb_tls_info;
1844 if_rele(tls->rx_ifp);
1846 tls->rx_ifp = ifp;
1852 mtx_pool_lock(mtxpool_sleep, tls);
1853 if (!tls->reset_pending) {
1854 (void) ktls_hold(tls);
1856 tls->so = so;
1857 tls->reset_pending = true;
1858 taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);
1860 mtx_pool_unlock(mtxpool_sleep, tls);
1864 ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls)
1876 mtx_pool_lock(mtxpool_sleep, tls);
1877 if (!tls->reset_pending) {
1878 (void) ktls_hold(tls);
1879 tls->reset_pending = true;
1880 taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);
1882 mtx_pool_unlock(mtxpool_sleep, tls);
1888 ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate)
1899 MPASS(tls->mode == TCP_TLS_MODE_IFNET);
1901 if (tls->snd_tag == NULL) {
1911 mst = tls->snd_tag;
1927 ktls_destroy(struct ktls_session *tls)
1933 MPASS(tls->refcount == 0);
1935 inp = tls->inp;
1936 if (tls->tx) {
1959 TASK_INIT(&tls->destroy_task, 0,
1960 ktls_destroy_help, tls);
1962 &tls->destroy_task);
1968 if (tls->sequential_records) {
1972 STAILQ_FOREACH_SAFE(m, &tls->pending_records, m_epg_stailq, n) {
1984 switch (tls->mode) {
1986 switch (tls->params.cipher_algorithm) {
1999 switch (tls->params.cipher_algorithm) {
2010 if (tls->snd_tag != NULL)
2011 m_snd_tag_rele(tls->snd_tag);
2012 if (tls->rx_ifp != NULL)
2013 if_rele(tls->rx_ifp);
2014 if (tls->tx) {
2023 switch (tls->params.cipher_algorithm) {
2037 if (tls->ocf_session != NULL)
2038 ktls_ocf_free(tls);
2039 if (tls->params.auth_key != NULL) {
2040 zfree(tls->params.auth_key, M_KTLS);
2041 tls->params.auth_key = NULL;
2042 tls->params.auth_key_len = 0;
2044 if (tls->params.cipher_key != NULL) {
2045 zfree(tls->params.cipher_key, M_KTLS);
2046 tls->params.cipher_key = NULL;
2047 tls->params.cipher_key_len = 0;
2049 if (tls->tx) {
2054 explicit_bzero(tls->params.iv, sizeof(tls->params.iv));
2056 uma_zfree(ktls_session_zone, tls);
2088 ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
2097 maxlen = tls->params.max_frame_len;
2107 (m->m_len > 0 || ktls_permit_empty_frames(tls)),
2120 m->m_epg_tls = ktls_hold(tls);
2122 m->m_epg_hdrlen = tls->params.tls_hlen;
2123 m->m_epg_trllen = tls->params.tls_tlen;
2124 if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) {
2137 * tls->params.tls_tlen is the maximum
2143 bs = tls->params.tls_bs;
2144 delta = (tls_len + tls->params.tls_tlen) & (bs - 1);
2151 tlshdr->tls_vmajor = tls->params.tls_vmajor;
2157 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE &&
2158 tls->params.tls_vmajor == TLS_MAJOR_VER_ONE) {
2165 tlshdr->tls_vminor = tls->params.tls_vminor;
2180 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 &&
2181 tls->params.tls_vminor == TLS_MINOR_VER_TWO) {
2182 noncep = (uint64_t *)(tls->params.iv + 8);
2185 } else if (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&
2186 tls->params.tls_vminor >= TLS_MINOR_VER_ONE)
2197 if (tls->mode == TCP_TLS_MODE_SW) {
2210 ktls_permit_empty_frames(struct ktls_session *tls)
2212 return (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&
2213 tls->params.tls_vminor == TLS_MINOR_VER_ZERO);
2373 tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len,
2397 if (last_offset < tls->params.tls_hlen)
2514 struct ktls_session *tls;
2531 tls = sb->sb_tls_info;
2532 MPASS(tls != NULL);
2534 tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE);
2538 vminor = tls->params.tls_vminor;
2541 if (sb->sb_tlscc < tls->params.tls_hlen)
2544 m_copydata(sb->sb_mtls, 0, tls->params.tls_hlen, tls_header);
2547 if (hdr->tls_vmajor != tls->params.tls_vmajor ||
2552 else if (tls_len < tls->params.tls_hlen || tls_len >
2553 tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 +
2554 tls->params.tls_tlen)
2594 error = ktls_ocf_recrypt(tls, hdr, data, seqno);
2599 error = ktls_ocf_decrypt(tls, hdr, data, seqno,
2603 error = tls13_find_record_type(tls, data,
2617 trail_len = tls->params.tls_tlen - 1;
2618 error = tls13_find_record_type(tls, data,
2621 trail_len = tls->params.tls_tlen;
2668 tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen -
2696 remain = tls->params.tls_hlen;
2812 struct ktls_session *tls, struct ktls_ocf_encrypt_state *state)
2825 return (ktls_ocf_encrypt(state, tls, m, NULL, 0));
2855 error = ktls_ocf_encrypt(state, tls, m, state->dst_iov, i + 1);
2892 struct ktls_session *tls;
2913 tls = m->m_epg_tls;
2914 wq = &ktls_wq[tls->wq_index];
2916 if (__predict_false(tls->sequential_records)) {
2926 * tls->next_seqno holds the sequence number of the
2928 * queue. If this next record is not tls->next_seqno,
2930 * TLS sequence number, into tls->pending_records and
2933 * If this TLS record matches tls->next_seqno, place
2935 * tls->pending_records to see if any
2939 if (m->m_epg_seqno != tls->next_seqno) {
2943 STAILQ_FOREACH(n, &tls->pending_records, m_epg_stailq) {
2949 STAILQ_INSERT_TAIL(&tls->pending_records, m,
2952 STAILQ_INSERT_HEAD(&tls->pending_records, m,
2955 STAILQ_INSERT_AFTER(&tls->pending_records, p, m,
2962 tls->next_seqno += ktls_batched_records(m);
2965 while (!STAILQ_EMPTY(&tls->pending_records)) {
2968 n = STAILQ_FIRST(&tls->pending_records);
2969 if (n->m_epg_seqno != tls->next_seqno)
2973 STAILQ_REMOVE_HEAD(&tls->pending_records, m_epg_stailq);
2974 tls->next_seqno += ktls_batched_records(n);
3026 struct ktls_session *tls;
3032 tls = top->m_epg_tls;
3033 KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));
3060 KASSERT(m->m_epg_tls == tls,
3062 tls, m->m_epg_tls));
3067 error = ktls_encrypt_record(wq, m, tls, &state);
3087 ktls_free(tls);
3105 struct ktls_session *tls;
3125 tls = m->m_epg_tls;
3127 ktls_free(tls);
3155 struct ktls_session *tls;
3161 tls = top->m_epg_tls;
3162 KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));
3172 KASSERT(m->m_epg_tls == tls,
3174 tls, m->m_epg_tls));
3187 error = ktls_encrypt_record(wq, m, tls, state);
3349 struct ktls_session *tls;
3355 tls = context;
3356 inp = tls->inp;
3386 ktls_free(tls);
3408 struct ktls_session *tls;
3415 tls = so->so_snd.sb_tls_info;
3427 (void)ktls_hold(tls);
3431 TASK_INIT(&tls->disable_ifnet_task, 0, ktls_disable_ifnet_help, tls);
3432 (void)taskqueue_enqueue(taskqueue_thread, &tls->disable_ifnet_task);