1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #include <rte_ip.h> 11 #include <rte_tcp.h> 12 13 #include "sfc.h" 14 #include "sfc_debug.h" 15 #include "sfc_tx.h" 16 #include "sfc_ev.h" 17 #include "sfc_tso.h" 18 19 int 20 sfc_efx_tso_alloc_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, 21 unsigned int txq_entries, unsigned int socket_id) 22 { 23 unsigned int i; 24 25 for (i = 0; i < txq_entries; ++i) { 26 sw_ring[i].tsoh = rte_malloc_socket("sfc-efx-txq-tsoh-obj", 27 SFC_TSOH_STD_LEN, 28 RTE_CACHE_LINE_SIZE, 29 socket_id); 30 if (sw_ring[i].tsoh == NULL) 31 goto fail_alloc_tsoh_objs; 32 } 33 34 return 0; 35 36 fail_alloc_tsoh_objs: 37 while (i > 0) 38 rte_free(sw_ring[--i].tsoh); 39 40 return ENOMEM; 41 } 42 43 void 44 sfc_efx_tso_free_tsoh_objs(struct sfc_efx_tx_sw_desc *sw_ring, 45 unsigned int txq_entries) 46 { 47 unsigned int i; 48 49 for (i = 0; i < txq_entries; ++i) { 50 rte_free(sw_ring[i].tsoh); 51 sw_ring[i].tsoh = NULL; 52 } 53 } 54 55 unsigned int 56 sfc_tso_prepare_header(uint8_t *tsoh, size_t header_len, 57 struct rte_mbuf **in_seg, size_t *in_off) 58 { 59 struct rte_mbuf *m = *in_seg; 60 size_t bytes_to_copy = 0; 61 size_t bytes_left = header_len; 62 unsigned int segments_copied = 0; 63 64 do { 65 bytes_to_copy = MIN(bytes_left, m->data_len); 66 67 rte_memcpy(tsoh, rte_pktmbuf_mtod(m, uint8_t *), 68 bytes_to_copy); 69 70 bytes_left -= bytes_to_copy; 71 tsoh += bytes_to_copy; 72 73 if (bytes_left > 0) { 74 m = m->next; 75 SFC_ASSERT(m != NULL); 76 segments_copied++; 77 } 78 } while (bytes_left > 0); 79 80 if (bytes_to_copy == m->data_len) { 81 *in_seg = m->next; 82 *in_off = 0; 83 segments_copied++; 84 } else { 85 *in_seg = m; 86 *in_off = bytes_to_copy; 87 } 88 89 return segments_copied; 90 } 91 92 int 93 sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx, 94 struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend, 95 unsigned int *pkt_descs, size_t *pkt_len) 96 { 97 uint8_t *tsoh; 98 const struct rte_tcp_hdr *th; 99 efsys_dma_addr_t header_paddr; 100 uint16_t packet_id = 0; 101 uint32_t sent_seq; 102 struct rte_mbuf *m = *in_seg; 103 size_t nh_off = m->l2_len; /* IP header offset */ 104 size_t tcph_off = m->l2_len + m->l3_len; /* TCP header offset */ 105 size_t header_len = m->l2_len + m->l3_len + m->l4_len; 106 107 idx += SFC_EF10_TSO_OPT_DESCS_NUM; 108 109 header_paddr = rte_pktmbuf_iova(m); 110 111 /* 112 * Sometimes headers may be split across multiple mbufs. In such cases 113 * we need to glue those pieces and store them in some temporary place. 114 * Also, packet headers must be contiguous in memory, so that 115 * they can be referred to with a single DMA descriptor. EF10 has no 116 * limitations on address boundaries crossing by DMA descriptor data. 117 */ 118 if (m->data_len < header_len) { 119 /* 120 * Discard a packet if header linearization is needed but 121 * the header is too big. 122 * Duplicate Tx prepare check here to avoid spoil of 123 * memory if Tx prepare is skipped. 124 */ 125 if (unlikely(header_len > SFC_TSOH_STD_LEN)) 126 return EMSGSIZE; 127 128 tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh; 129 sfc_tso_prepare_header(tsoh, header_len, in_seg, in_off); 130 131 header_paddr = rte_malloc_virt2iova((void *)tsoh); 132 } else { 133 if (m->data_len == header_len) { 134 *in_off = 0; 135 *in_seg = m->next; 136 } else { 137 *in_off = header_len; 138 } 139 140 tsoh = rte_pktmbuf_mtod(m, uint8_t *); 141 } 142 143 /* 144 * 8000-series EF10 hardware requires that innermost IP length 145 * be greater than or equal to the value which each segment is 146 * supposed to have; otherwise, TCP checksum will be incorrect. 147 */ 148 sfc_tso_innermost_ip_fix_len(m, tsoh, nh_off); 149 150 /* 151 * Handle IP header. Tx prepare has debug-only checks that offload flags 152 * are correctly filled in in TSO mbuf. Use zero IPID if there is no 153 * IPv4 flag. If the packet is still IPv4, HW will simply start from 154 * zero IPID. 155 */ 156 if (m->ol_flags & PKT_TX_IPV4) 157 packet_id = sfc_tso_ip4_get_ipid(tsoh, nh_off); 158 159 /* Handle TCP header */ 160 th = (const struct rte_tcp_hdr *)(tsoh + tcph_off); 161 162 rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t)); 163 sent_seq = rte_be_to_cpu_32(sent_seq); 164 165 efx_tx_qdesc_tso2_create(txq->common, packet_id, 0, sent_seq, 166 m->tso_segsz, 167 *pend, EFX_TX_FATSOV2_OPT_NDESCS); 168 169 *pend += EFX_TX_FATSOV2_OPT_NDESCS; 170 *pkt_descs += EFX_TX_FATSOV2_OPT_NDESCS; 171 172 efx_tx_qdesc_dma_create(txq->common, header_paddr, header_len, 173 B_FALSE, (*pend)++); 174 (*pkt_descs)++; 175 *pkt_len -= header_len; 176 177 return 0; 178 } 179