Lines Matching +full:wr +full:- +full:setup
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2018-2019 Chelsio Communications, Inc.
91 /* Only used outside of setup and teardown when using inline keys. */
94 /* Fields only used during setup and teardown. */
120 struct port_info *pi = vi->pi;
121 struct adapter *sc = pi->adapter;
128 m_snd_tag_init(&tlsp->com, ifp, &t6_tls_tag_sw);
129 tlsp->vi = vi;
130 tlsp->sc = sc;
131 tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id];
132 tlsp->tid = -1;
133 tlsp->tx_key_addr = -1;
161 qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
165 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
166 &cpl->peer_ip, &cpl->peer_port);
169 options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan);
171 cpl->opt0 = htobe64(options);
173 options = V_TX_QUEUE(TX_MODQ(vi->pi->tx_chan));
174 if (tp->t_flags & TF_REQ_TSTMP)
176 cpl->opt2 = htobe32(options);
192 qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
196 cpl->local_port = inp->inp_lport;
197 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
198 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
199 cpl->peer_port = inp->inp_fport;
200 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
201 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
204 options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan);
206 cpl->opt0 = htobe64(options);
208 options = V_TX_QUEUE(TX_MODQ(vi->pi->tx_chan));
209 if (tp->t_flags & TF_REQ_TSTMP)
211 cpl->opt2 = htobe32(options);
218 struct wrqe *wr;
221 isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
223 tlsp->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
224 if (tlsp->ce == NULL)
228 wr = alloc_wrqe(ktls_act_open_cpl_size(isipv6), tlsp->ctrlq);
229 if (wr == NULL) {
230 CTR2(KTR_CXGBE, "%s: atid %d failed to alloc WR", __func__,
236 mk_ktls_act_open_req6(sc, vi, inp, tlsp, atid, wrtod(wr));
238 mk_ktls_act_open_req(sc, vi, inp, tlsp, atid, wrtod(wr));
240 tlsp->open_pending = true;
241 t4_wrq_tx(sc, wr);
249 struct adapter *sc = iq->adapter;
251 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
252 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
254 struct inpcb *inp = tlsp->inp;
259 tlsp->tid = GET_TID(cpl);
262 tlsp->open_pending = false;
273 "CPL_SET_TCB_FIELD ULP command not 16-byte aligned");
285 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
287 V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) |
288 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1));
289 txpkt->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
291 /* ULPTX_IDATA sub-command */
293 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
294 idata->len = htobe32(sizeof(*cpl));
298 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tlsp->tid));
299 cpl->reply_ctrl = htobe16(F_NO_REPLY);
300 cpl->word_cookie = htobe16(V_WORD(word));
301 cpl->mask = htobe64(mask);
302 cpl->val = htobe64(val);
306 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
307 idata->len = htobe32(0);
313 struct fw_ulptx_wr *wr;
319 len = sizeof(*wr) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16);
320 if (tp->t_flags & TF_REQ_TSTMP)
324 CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__,
325 tlsp->tid);
328 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);
329 m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
332 wr = mtod(m, void *);
333 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
334 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
336 wr->cookie = 0;
337 dst = (char *)(wr + 1);
356 if (tp->t_flags & TF_REQ_TSTMP) {
359 V_TCB_TIMESTAMP_OFFSET(tp->ts_offset >> 28));
363 KASSERT(dst - (char *)wr == len, ("%s: length mismatch", __func__));
366 error = mp_ring_enqueue(txq->r, items, 1, 1);
385 tls = params->tls.tls;
388 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
389 tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
390 tls->params.tls_vminor > TLS_MINOR_VER_TWO)
394 switch (tls->params.cipher_algorithm) {
397 switch (tls->params.cipher_key_len) {
405 switch (tls->params.auth_algorithm) {
417 if (tls->params.iv_len != SALT_SIZE)
419 switch (tls->params.cipher_key_len) {
435 sc = vi->adapter;
445 if (sc->tlst.inline_keys)
446 keyid = -1;
452 tlsp->inline_key = true;
454 tlsp->tx_key_addr = keyid;
457 atid, tlsp->tx_key_addr);
460 inp = params->tls.inp;
462 if (inp->inp_flags & INP_DROPPED) {
467 tlsp->inp = inp;
470 if (tp->t_flags & TF_REQ_TSTMP) {
471 tlsp->using_timestamps = true;
472 if ((tp->ts_offset & 0xfffffff) != 0) {
478 tlsp->using_timestamps = false;
489 while (tlsp->open_pending) {
494 error = rw_sleep(tlsp, &inp->inp_lock, 0, "t6tlsop", 0);
497 atid = -1;
498 if (tlsp->tid < 0) {
504 if (inp->inp_flags & INP_DROPPED) {
510 txq = &sc->sge.txq[vi->first_txq];
511 if (inp->inp_flowtype != M_HASHTYPE_NONE)
512 txq += ((inp->inp_flowid % (vi->ntxq - vi->rsrv_noflowq)) +
513 vi->rsrv_noflowq);
514 tlsp->txq = txq;
525 tlsp->enc_mode = t4_tls_cipher_mode(tls);
526 tlsp->tx_key_info_size = t4_tls_key_info_size(tls);
529 tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) |
533 V_SCMD_CIPH_MODE(tlsp->enc_mode) |
538 tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
540 if (tlsp->inline_key)
541 tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);
542 tlsp->scmd0.ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen);
548 tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) |
555 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
556 tlsp->scmd0_short.seqno_numivs |=
559 tlsp->scmd0_short.seqno_numivs |=
560 V_SCMD_CIPH_MODE(tlsp->enc_mode);
561 tlsp->scmd0_short.seqno_numivs =
562 htobe32(tlsp->scmd0_short.seqno_numivs);
564 tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
567 if (tlsp->inline_key)
568 tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);
571 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
572 txq->kern_tls_gcm++;
574 txq->kern_tls_cbc++;
576 *pt = &tlsp->com;
582 m_snd_tag_rele(&tlsp->com);
603 t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx);
604 if (tlsp->inline_key)
610 CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__,
611 tlsp->tid);
614 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);
615 m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
619 t4_write_tlskey_wr(tls, KTLS_TX, tlsp->tid, 0, tlsp->tx_key_addr, kwr);
621 memcpy(kctx, &tlsp->keyctx, sizeof(*kctx));
629 error = mp_ring_enqueue(txq->r, items, 1, 1);
633 CTR2(KTR_CXGBE, "%s: tid %d sent key WR", __func__, tlsp->tid);
646 if (tlsp->inline_key)
647 wr_len += tlsp->tx_key_info_size;
664 hdr = (void *)m_tls->m_epg_hdr;
665 plen = ntohs(hdr->tls_length);
671 mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
682 if (mlen > TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen)
683 mlen = TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen;
687 * For AES-CBC adjust the ciphertext length for the block
690 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&
692 mlen = TLS_HEADER_LENGTH + rounddown(mlen - TLS_HEADER_LENGTH,
698 __func__, tlsp->tid, mlen, TLS_HEADER_LENGTH + plen);
706 * a non-zero offset implies that a header will not be sent.
718 hdr = (void *)m_tls->m_epg_hdr;
719 plen = ntohs(hdr->tls_length);
721 mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
724 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen)
726 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
735 offset = min(mtod(m_tls, vm_offset_t) - m_tls->m_epg_hdrlen,
736 (plen - TLS_HEADER_LENGTH - m_tls->m_epg_trllen) - 1);
748 nsegs--;
769 if (tlen <= m_tls->m_epg_hdrlen) {
776 roundup2(m->m_len + m_tls->m_len, 16);
779 "%s: tid %d TLS header-only packet too long (len %d)",
780 __func__, tlsp->tid, m->m_len + m_tls->m_len);
784 MPASS(m_tls->m_next == NULL);
789 hdr = (void *)m_tls->m_epg_hdr;
790 plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen;
807 imm_len += m_tls->m_epg_hdrlen;
813 *nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset,
814 plen - (m_tls->m_epg_hdrlen + offset));
822 * See if we have any TCP options requiring a dedicated options-only
832 cnt = tcp->th_off * 4 - sizeof(struct tcphdr);
833 for (; cnt > 0; cnt -= optlen, cp += optlen) {
867 cnt = tcp->th_off * 4 - sizeof(struct tcphdr);
868 for (; cnt > 0; cnt -= optlen, cp += optlen) {
908 MPASS(m->m_pkthdr.snd_tag != NULL);
909 tlsp = mst_to_tls(m->m_pkthdr.snd_tag);
911 if (m->m_len <= sizeof(*eh) + sizeof(*ip)) {
913 tlsp->tid);
917 if (ntohs(eh->ether_type) != ETHERTYPE_IP &&
918 ntohs(eh->ether_type) != ETHERTYPE_IPV6) {
920 __func__, tlsp->tid);
923 m->m_pkthdr.l2hlen = sizeof(*eh);
926 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
928 if (ip->ip_p != IPPROTO_TCP) {
930 __func__, tlsp->tid);
933 m->m_pkthdr.l3hlen = ip->ip_hl * 4;
936 if (ip6->ip6_nxt != IPPROTO_TCP) {
938 __func__, tlsp->tid, ip6->ip6_nxt);
941 m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr);
943 if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
946 __func__, tlsp->tid);
949 tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen);
950 m->m_pkthdr.l4hlen = tcp->th_off * 4;
953 if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
954 m->m_pkthdr.l4hlen) {
957 __func__, tlsp->tid, m->m_pkthdr.l2hlen,
958 m->m_pkthdr.l3hlen, m->m_pkthdr.l4hlen, m->m_len);
963 MPASS(m->m_next != NULL);
964 MPASS(m->m_next->m_flags & M_EXTPG);
972 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
973 MPASS(m_tls->m_flags & M_EXTPG);
978 tlsp->tid, wr_len, nsegs);
988 if (m_tls == m->m_next)
1000 sizeof(struct cpl_tx_pkt_core) + roundup2(m->m_len, 16);
1003 "%s: tid %d options-only packet too long (len %d)",
1004 __func__, tlsp->tid, m->m_len);
1025 if (tlsp->using_timestamps)
1033 tlsp->tid, mbuf_len16(m), mbuf_nsegs(m));
1036 return (mp_ring_enqueue(tlsp->txq->r, items, 1, 256));
1054 nsegs = gl->sg_nseg;
1057 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
1059 seg = &gl->sg_segs[0];
1062 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
1064 usgl->len0 = htobe32(seg->ss_len);
1065 usgl->addr0 = htobe64(seg->ss_paddr);
1068 for (i = 0; i < nsegs - 1; i++, seg++) {
1069 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
1070 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
1073 usgl->sge[i / 2].len[1] = htobe32(0);
1088 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
1089 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
1092 (uintptr_t)&eq->desc[eq->sidx])) {
1095 if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx])
1096 (*to) = (caddr_t)eq->desc;
1098 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
1102 portion = len - portion; /* remaining */
1103 bcopy(from, (void *)eq->desc, portion);
1104 (*to) = (caddr_t)eq->desc + portion;
1113 struct fw_eth_tx_pkt_wr *wr;
1127 wr = dst;
1128 pktlen = m->m_len;
1135 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
1139 wr->equiq_to_len16 = htobe32(ctrl);
1140 wr->r3 = 0;
1142 cpl = (void *)(wr + 1);
1145 cpl->ctrl0 = txq->cpl_ctrl0;
1146 cpl->pack = 0;
1147 cpl->len = htobe16(pktlen);
1153 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
1156 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
1157 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1159 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
1160 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
1161 if (m->m_pkthdr.l3hlen > sizeof(*ip))
1162 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
1163 m->m_pkthdr.l3hlen - sizeof(*ip));
1165 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1166 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1168 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1170 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen);
1171 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
1172 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
1174 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1175 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1177 cpl->ctrl1 = htobe64(ctrl1);
1178 txq->txcsum++;
1181 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1184 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
1187 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, pktlen -
1188 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
1189 txq->imm_wrs++;
1191 txq->txpkt_wrs++;
1193 txq->kern_tls_options++;
1195 txsd = &txq->sdesc[pidx];
1196 txsd->m = NULL;
1197 txsd->desc_used = ndesc;
1207 struct fw_eth_tx_pkt_wr *wr;
1225 MPASS(m_tls->m_next == NULL);
1227 wr = dst;
1228 pktlen = m->m_len + m_tls->m_len;
1235 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
1239 wr->equiq_to_len16 = htobe32(ctrl);
1240 wr->r3 = 0;
1242 cpl = (void *)(wr + 1);
1245 cpl->ctrl0 = txq->cpl_ctrl0;
1246 cpl->pack = 0;
1247 cpl->len = htobe16(pktlen);
1253 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
1256 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
1257 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1259 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
1260 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
1261 if (m->m_pkthdr.l3hlen > sizeof(*ip))
1262 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
1263 m->m_pkthdr.l3hlen - sizeof(*ip));
1265 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1266 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1268 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1270 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen);
1271 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
1272 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
1274 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1275 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1277 cpl->ctrl1 = htobe64(ctrl1);
1278 txq->txcsum++;
1281 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1284 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
1287 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len -
1288 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
1291 copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
1292 mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
1293 txq->imm_wrs++;
1295 txq->txpkt_wrs++;
1297 txq->kern_tls_header++;
1299 txsd = &txq->sdesc[pidx];
1300 txsd->m = m;
1301 txsd->desc_used = ndesc;
1316 struct sge_eq *eq = &txq->eq;
1318 struct fw_ulptx_wr *wr;
1334 MPASS(tlsp->txq == txq);
1336 first_wr = (tlsp->prev_seq == 0 && tlsp->prev_ack == 0 &&
1337 tlsp->prev_win == 0);
1340 * Use the per-txq scratch pad if near the end of the ring to
1341 * simplify handling of wrap-around. This uses a simple but
1345 using_scratch = (eq->sidx - pidx < SGE_MAX_WR_LEN / EQ_ESIZE);
1349 hdr = (void *)m_tls->m_epg_hdr;
1350 plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen;
1354 if (tlen <= m_tls->m_epg_hdrlen) {
1360 CTR3(KTR_CXGBE, "%s: tid %d header-only TLS record %u",
1361 __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno);
1371 __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno, offset);
1373 if (m_tls->m_next == NULL && (tcp_get_flags(tcp) & TH_FIN) != 0) {
1374 txq->kern_tls_fin_short++;
1388 last_wr = m_tls->m_next == NULL && (tcp_get_flags(tcp) & TH_FIN) == 0;
1400 * into MSS-sized chunks if the entire sequence range of those
1402 * TX_MAX - SND_UNA_RAW. Thus, use the offset stored in
1425 if (tx_max_offset > TLS_HEADER_LENGTH + ntohs(hdr->tls_length) -
1426 m_tls->m_epg_trllen) {
1428 tx_max_offset = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) -
1429 m_tls->m_epg_trllen;
1431 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&
1435 rounddown(tx_max_offset - TLS_HEADER_LENGTH,
1446 wr = (void *)txq->ss;
1448 wr = dst;
1449 out = (void *)(wr + 1);
1452 KASSERT(m->m_next == m_tls,
1453 ("trying to set L2T_IX for subsequent TLS WR"));
1456 tlsp->tid, tlsp->l2te->idx);
1459 V_TCB_L2T_IX(M_TCB_L2T_IX), V_TCB_L2T_IX(tlsp->l2te->idx));
1463 if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) {
1464 KASSERT(m->m_next == m_tls,
1465 ("trying to set T_RTSEQ_RECENT for subsequent TLS WR"));
1468 __func__, tlsp->tid);
1476 tlsp->prev_tsecr = ntohl(tsopt[1]);
1479 if (first_wr || tlsp->prev_seq != tx_max) {
1480 KASSERT(m->m_next == m_tls,
1481 ("trying to set TX_MAX for subsequent TLS WR"));
1485 __func__, tlsp->tid, tx_max, tcp_seqno);
1498 if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) {
1499 KASSERT(m->m_next == m_tls,
1500 ("trying to clear SND_UNA_RAW for subsequent TLS WR"));
1503 tlsp->tid);
1516 tlsp->prev_seq = tcp_seqno + tlen;
1518 if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) {
1519 KASSERT(m->m_next == m_tls,
1520 ("trying to set RCV_NXT for subsequent TLS WR"));
1523 V_TCB_RCV_NXT(ntohl(tcp->th_ack)));
1527 tlsp->prev_ack = ntohl(tcp->th_ack);
1530 if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) {
1531 KASSERT(m->m_next == m_tls,
1532 ("trying to set RCV_WND for subsequent TLS WR"));
1535 V_TCB_RCV_WND(ntohs(tcp->th_win)));
1539 tlsp->prev_win = ntohs(tcp->th_win);
1543 if (m->m_next == m_tls)
1546 nsegs = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen +
1547 offset, plen - (m_tls->m_epg_hdrlen + offset));
1554 imm_len += m_tls->m_epg_hdrlen;
1563 * FW_ULPTX_WR work request header at 'wr' as a dedicated work
1570 tlsp->sc->tlst.combo_wrs) {
1574 wr_len += sizeof(*wr);
1575 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
1576 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
1578 wr->cookie = 0;
1586 copy_to_txd(eq, txq->ss, &out, wr_len);
1592 txq->raw_wrs++;
1593 txsd = &txq->sdesc[pidx];
1594 txsd->m = NULL;
1595 txsd->desc_used = ndesc;
1596 IDXINCR(pidx, ndesc, eq->sidx);
1597 dst = &eq->desc[pidx];
1606 using_scratch = (eq->sidx - pidx <
1609 wr = (void *)txq->ss;
1611 wr = dst;
1612 txpkt = (void *)(wr + 1);
1623 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
1624 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
1626 wr->cookie = 0;
1629 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
1631 V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) |
1632 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1));
1633 txpkt->len = htobe32(howmany(twr_len - sizeof(*wr), 16));
1635 /* ULPTX_IDATA sub-command */
1637 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1639 idata->len = sizeof(struct cpl_tx_sec_pdu);
1647 if (tlsp->inline_key)
1648 idata->len += tlsp->tx_key_info_size +
1650 idata->len = htobe32(idata->len);
1669 sec_pdu->pldlen = htobe32(16 + plen -
1670 (m_tls->m_epg_hdrlen + offset));
1673 sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs;
1674 sec_pdu->ivgen_hdrlen = htobe32(
1675 tlsp->scmd0_short.ivgen_hdrlen |
1676 V_SCMD_HDR_LEN(offset == 0 ? m_tls->m_epg_hdrlen : 0));
1678 txq->kern_tls_short++;
1689 cipher_start = m_tls->m_epg_hdrlen + 1;
1690 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
1702 sec_pdu->pldlen = htobe32(plen);
1705 sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs;
1706 sec_pdu->ivgen_hdrlen = tlsp->scmd0.ivgen_hdrlen;
1709 txq->kern_tls_full++;
1711 txq->kern_tls_partial++;
1713 sec_pdu->op_ivinsrtofst = htobe32(
1717 sec_pdu->aadstart_cipherstop_hi = htobe32(
1722 sec_pdu->cipherstop_lo_authinsert = htobe32(
1728 sec_pdu->scmd1 = htobe64(m_tls->m_epg_seqno);
1732 if (tlsp->inline_key) {
1733 memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size);
1734 out += tlsp->tx_key_info_size;
1738 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
1740 V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4));
1741 memrd->addr = htobe32(tlsp->tx_key_addr >> 5);
1745 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1747 idata->len = htobe32(sizeof(struct cpl_tx_data) + imm_len);
1754 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tlsp->tid));
1755 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1756 mss = m->m_pkthdr.tso_segsz;
1757 tlsp->prev_mss = mss;
1758 } else if (tlsp->prev_mss != 0)
1759 mss = tlsp->prev_mss;
1761 mss = if_getmtu(tlsp->vi->ifp) -
1762 (m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen);
1764 tx_data->len = htobe32(V_TX_DATA_MSS(mss) | V_TX_LENGTH(tlen));
1765 tx_data->rsvd = htobe32(tcp_seqno);
1767 tx_data->len = htobe32(V_TX_DATA_MSS(mss) |
1768 V_TX_LENGTH(tlen - (m_tls->m_epg_hdrlen + offset)));
1769 tx_data->rsvd = htobe32(tcp_seqno + m_tls->m_epg_hdrlen + offset);
1771 tx_data->flags = htobe32(F_TX_BYPASS);
1773 tx_data->flags |= htobe32(F_TX_PUSH | F_TX_SHOVE);
1778 memcpy(out, m_tls->m_epg_hdr, m_tls->m_epg_hdrlen);
1779 out += m_tls->m_epg_hdrlen;
1785 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
1786 memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE);
1796 /* Zero pad to an 8-byte boundary. */
1797 memset(out, 0, 8 - (imm_len % 8));
1798 out += 8 - (imm_len % 8);
1802 * 16-byte aligned.
1806 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
1807 idata->len = htobe32(0);
1813 sglist_reset(txq->gl);
1814 if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_epg_hdrlen + offset,
1815 plen - (m_tls->m_epg_hdrlen + offset)) != 0) {
1820 write_gl_to_buf(txq->gl, out);
1824 copy_to_txd(eq, txq->ss, &out, wr_len);
1830 txq->kern_tls_records++;
1831 txq->kern_tls_octets += tlen - mtod(m_tls, vm_offset_t);
1834 txq->kern_tls_waste += mtod(m_tls, vm_offset_t);
1836 txq->kern_tls_waste += mtod(m_tls, vm_offset_t) -
1837 (m_tls->m_epg_hdrlen + offset);
1840 txsd = &txq->sdesc[pidx];
1842 txsd->m = m;
1844 txsd->m = NULL;
1845 txsd->desc_used = howmany(wr_len, EQ_ESIZE);
1855 struct fw_eth_tx_pkt_wr *wr;
1869 wr = dst;
1870 pktlen = m->m_len;
1877 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
1881 wr->equiq_to_len16 = htobe32(ctrl);
1882 wr->r3 = 0;
1884 cpl = (void *)(wr + 1);
1887 cpl->ctrl0 = txq->cpl_ctrl0;
1888 cpl->pack = 0;
1889 cpl->len = htobe16(pktlen);
1895 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
1898 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
1899 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1901 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
1902 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
1903 if (m->m_pkthdr.l3hlen > sizeof(*ip))
1904 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
1905 m->m_pkthdr.l3hlen - sizeof(*ip));
1907 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1908 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1910 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1912 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen);
1913 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
1914 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
1916 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1917 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1919 cpl->ctrl1 = htobe64(ctrl1);
1920 txq->txcsum++;
1923 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1926 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
1929 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len -
1930 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
1931 txq->imm_wrs++;
1933 txq->txpkt_wrs++;
1935 txq->kern_tls_fin++;
1937 txsd = &txq->sdesc[pidx];
1938 txsd->m = m;
1939 txsd->desc_used = ndesc;
1948 struct sge_eq *eq = &txq->eq;
1961 MPASS(m->m_pkthdr.snd_tag != NULL);
1962 tlsp = mst_to_tls(m->m_pkthdr.snd_tag);
1966 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen +
1967 m->m_pkthdr.l3hlen);
1968 pidx = eq->pidx;
1978 IDXINCR(pidx, ndesc, eq->sidx);
1979 dst = &eq->desc[pidx];
1982 tlsp->tid);
1990 if (m->m_flags & M_VLANTAG)
1991 vlan_tag = m->m_pkthdr.ether_vtag;
1995 if (tlsp->l2te == NULL || tlsp->l2te->vlan != vlan_tag ||
1996 memcmp(tlsp->l2te->dmac, eh->ether_dhost, ETHER_ADDR_LEN) != 0) {
1998 if (tlsp->l2te)
1999 t4_l2t_release(tlsp->l2te);
2000 tlsp->l2te = t4_l2t_alloc_tls(tlsp->sc, txq, dst, &ndesc,
2001 vlan_tag, tlsp->vi->pi->lport, eh->ether_dhost);
2002 if (tlsp->l2te == NULL)
2005 MPASS(ndesc <= available - totdesc);
2007 txq->raw_wrs++;
2008 txsd = &txq->sdesc[pidx];
2009 txsd->m = NULL;
2010 txsd->desc_used = ndesc;
2012 IDXINCR(pidx, ndesc, eq->sidx);
2013 dst = &eq->desc[pidx];
2021 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
2022 MPASS(m_tls->m_flags & M_EXTPG);
2029 if (m_tls == m->m_next) {
2030 tcp_seqno = ntohl(tcp->th_seq) -
2032 if (tlsp->using_timestamps)
2036 tcp_seqno = tlsp->prev_seq;
2040 available - totdesc, tcp_seqno, tsopt, pidx, set_l2t_idx);
2042 IDXINCR(pidx, ndesc, eq->sidx);
2043 dst = &eq->desc[pidx];
2059 tlsp->prev_seq, pidx);
2074 sc = tlsp->sc;
2076 CTR2(KTR_CXGBE, "%s: tid %d", __func__, tlsp->tid);
2078 if (tlsp->l2te)
2079 t4_l2t_release(tlsp->l2te);
2080 if (tlsp->tid >= 0)
2081 release_tid(sc, tlsp->tid, tlsp->ctrlq);
2082 if (tlsp->ce)
2083 t4_release_clip_entry(sc, tlsp->ce);
2084 if (tlsp->tx_key_addr >= 0)
2085 t4_free_tls_keyid(sc, tlsp->tx_key_addr);