1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 #ifndef _SFC_DP_TX_H 11 #define _SFC_DP_TX_H 12 13 #include <ethdev_driver.h> 14 15 #include "sfc_dp.h" 16 #include "sfc_debug.h" 17 #include "sfc_tso.h" 18 19 #ifdef __cplusplus 20 extern "C" { 21 #endif 22 23 /** 24 * Generic transmit queue information used on data path. 25 * It must be kept as small as it is possible since it is built into 26 * the structure used on datapath. 27 */ 28 struct sfc_dp_txq { 29 struct sfc_dp_queue dpq; 30 }; 31 32 /** Datapath transmit queue descriptor number limitations */ 33 struct sfc_dp_tx_hw_limits { 34 unsigned int txq_max_entries; 35 unsigned int txq_min_entries; 36 }; 37 38 /** 39 * Datapath transmit queue creation information. 40 * 41 * The structure is used just to pass information from control path to 42 * datapath. It could be just function arguments, but it would be hardly 43 * readable. 44 */ 45 struct sfc_dp_tx_qcreate_info { 46 /** Maximum number of pushed Tx descriptors */ 47 unsigned int max_fill_level; 48 /** Minimum number of unused Tx descriptors to do reap */ 49 unsigned int free_thresh; 50 /** Offloads enabled on the transmit queue */ 51 uint64_t offloads; 52 /** Tx queue size */ 53 unsigned int txq_entries; 54 /** Maximum size of data in the DMA descriptor */ 55 uint16_t dma_desc_size_max; 56 /** DMA-mapped Tx descriptors ring */ 57 void *txq_hw_ring; 58 /** Associated event queue size */ 59 unsigned int evq_entries; 60 /** Hardware event ring */ 61 void *evq_hw_ring; 62 /** The queue index in hardware (required to push right doorbell) */ 63 unsigned int hw_index; 64 /** Virtual address of the memory-mapped BAR to push Tx doorbell */ 65 volatile void *mem_bar; 66 /** VI window size shift */ 67 unsigned int vi_window_shift; 68 /** 69 * Maximum number of bytes into the packet the TCP header can start for 70 * the hardware to apply TSO packet edits. 71 */ 72 uint16_t tso_tcp_header_offset_limit; 73 /** Maximum number of header DMA descriptors per TSOv3 transaction */ 74 uint16_t tso_max_nb_header_descs; 75 /** Maximum header length acceptable by TSOv3 transaction */ 76 uint16_t tso_max_header_len; 77 /** Maximum number of payload DMA descriptors per TSOv3 transaction */ 78 uint16_t tso_max_nb_payload_descs; 79 /** Maximum payload length per TSOv3 transaction */ 80 uint32_t tso_max_payload_len; 81 /** Maximum number of frames to be generated per TSOv3 transaction */ 82 uint32_t tso_max_nb_outgoing_frames; 83 }; 84 85 /** 86 * Get Tx datapath specific device info. 87 * 88 * @param dev_info Device info to be adjusted 89 */ 90 typedef void (sfc_dp_tx_get_dev_info_t)(struct rte_eth_dev_info *dev_info); 91 92 /** 93 * Get size of transmit and event queue rings by the number of Tx 94 * descriptors. 95 * 96 * @param nb_tx_desc Number of Tx descriptors 97 * @param txq_entries Location for number of Tx ring entries 98 * @param evq_entries Location for number of event ring entries 99 * @param txq_max_fill_level Location for maximum Tx ring fill level 100 * 101 * @return 0 or positive errno. 102 */ 103 typedef int (sfc_dp_tx_qsize_up_rings_t)(uint16_t nb_tx_desc, 104 struct sfc_dp_tx_hw_limits *limits, 105 unsigned int *txq_entries, 106 unsigned int *evq_entries, 107 unsigned int *txq_max_fill_level); 108 109 /** 110 * Allocate and initialize datapath transmit queue. 111 * 112 * @param port_id The port identifier 113 * @param queue_id The queue identifier 114 * @param pci_addr PCI function address 115 * @param socket_id Socket identifier to allocate memory 116 * @param info Tx queue details wrapped in structure 117 * @param dp_txqp Location for generic datapath transmit queue pointer 118 * 119 * @return 0 or positive errno. 120 */ 121 typedef int (sfc_dp_tx_qcreate_t)(uint16_t port_id, uint16_t queue_id, 122 const struct rte_pci_addr *pci_addr, 123 int socket_id, 124 const struct sfc_dp_tx_qcreate_info *info, 125 struct sfc_dp_txq **dp_txqp); 126 127 /** 128 * Free resources allocated for datapath transmit queue. 129 */ 130 typedef void (sfc_dp_tx_qdestroy_t)(struct sfc_dp_txq *dp_txq); 131 132 /** 133 * Transmit queue start callback. 134 * 135 * It handovers EvQ to the datapath. 136 */ 137 typedef int (sfc_dp_tx_qstart_t)(struct sfc_dp_txq *dp_txq, 138 unsigned int evq_read_ptr, 139 unsigned int txq_desc_index); 140 141 /** 142 * Transmit queue stop function called before the queue flush. 143 * 144 * It returns EvQ to the control path. 145 */ 146 typedef void (sfc_dp_tx_qstop_t)(struct sfc_dp_txq *dp_txq, 147 unsigned int *evq_read_ptr); 148 149 /** 150 * Transmit event handler used during queue flush only. 151 */ 152 typedef bool (sfc_dp_tx_qtx_ev_t)(struct sfc_dp_txq *dp_txq, unsigned int id); 153 154 /** 155 * Transmit queue function called after the queue flush. 156 */ 157 typedef void (sfc_dp_tx_qreap_t)(struct sfc_dp_txq *dp_txq); 158 159 /** 160 * Check Tx descriptor status 161 */ 162 typedef int (sfc_dp_tx_qdesc_status_t)(struct sfc_dp_txq *dp_txq, 163 uint16_t offset); 164 165 /** Transmit datapath definition */ 166 struct sfc_dp_tx { 167 struct sfc_dp dp; 168 169 unsigned int features; 170 #define SFC_DP_TX_FEAT_MULTI_PROCESS 0x1 171 /** 172 * Tx offload capabilities supported by the datapath on device 173 * level only if HW/FW supports it. 174 */ 175 uint64_t dev_offload_capa; 176 /** 177 * Tx offload capabilities supported by the datapath per-queue 178 * if HW/FW supports it. 179 */ 180 uint64_t queue_offload_capa; 181 sfc_dp_tx_get_dev_info_t *get_dev_info; 182 sfc_dp_tx_qsize_up_rings_t *qsize_up_rings; 183 sfc_dp_tx_qcreate_t *qcreate; 184 sfc_dp_tx_qdestroy_t *qdestroy; 185 sfc_dp_tx_qstart_t *qstart; 186 sfc_dp_tx_qstop_t *qstop; 187 sfc_dp_tx_qtx_ev_t *qtx_ev; 188 sfc_dp_tx_qreap_t *qreap; 189 sfc_dp_tx_qdesc_status_t *qdesc_status; 190 eth_tx_prep_t pkt_prepare; 191 eth_tx_burst_t pkt_burst; 192 }; 193 194 static inline struct sfc_dp_tx * 195 sfc_dp_find_tx_by_name(struct sfc_dp_list *head, const char *name) 196 { 197 struct sfc_dp *p = sfc_dp_find_by_name(head, SFC_DP_TX, name); 198 199 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); 200 } 201 202 static inline struct sfc_dp_tx * 203 sfc_dp_find_tx_by_caps(struct sfc_dp_list *head, unsigned int avail_caps) 204 { 205 struct sfc_dp *p = sfc_dp_find_by_caps(head, SFC_DP_TX, avail_caps); 206 207 return (p == NULL) ? NULL : container_of(p, struct sfc_dp_tx, dp); 208 } 209 210 /** Get Tx datapath ops by the datapath TxQ handle */ 211 const struct sfc_dp_tx *sfc_dp_tx_by_dp_txq(const struct sfc_dp_txq *dp_txq); 212 213 static inline uint64_t 214 sfc_dp_tx_offload_capa(const struct sfc_dp_tx *dp_tx) 215 { 216 return dp_tx->dev_offload_capa | dp_tx->queue_offload_capa; 217 } 218 219 static inline unsigned int 220 sfc_dp_tx_pkt_extra_hdr_segs(struct rte_mbuf **m_seg, 221 unsigned int *header_len_remaining) 222 { 223 unsigned int nb_extra_header_segs = 0; 224 225 while (rte_pktmbuf_data_len(*m_seg) < *header_len_remaining) { 226 *header_len_remaining -= rte_pktmbuf_data_len(*m_seg); 227 *m_seg = (*m_seg)->next; 228 ++nb_extra_header_segs; 229 } 230 231 return nb_extra_header_segs; 232 } 233 234 static inline int 235 sfc_dp_tx_prepare_pkt(struct rte_mbuf *m, 236 unsigned int max_nb_header_segs, 237 unsigned int tso_bounce_buffer_len, 238 uint32_t tso_tcp_header_offset_limit, 239 unsigned int max_fill_level, 240 unsigned int nb_tso_descs, 241 unsigned int nb_vlan_descs) 242 { 243 unsigned int descs_required = m->nb_segs; 244 unsigned int tcph_off = ((m->ol_flags & PKT_TX_TUNNEL_MASK) ? 245 m->outer_l2_len + m->outer_l3_len : 0) + 246 m->l2_len + m->l3_len; 247 unsigned int header_len = tcph_off + m->l4_len; 248 unsigned int header_len_remaining = header_len; 249 unsigned int nb_header_segs = 1; 250 struct rte_mbuf *m_seg = m; 251 252 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG 253 int ret; 254 255 ret = rte_validate_tx_offload(m); 256 if (ret != 0) { 257 /* 258 * Negative error code is returned by rte_validate_tx_offload(), 259 * but positive are used inside net/sfc PMD. 260 */ 261 SFC_ASSERT(ret < 0); 262 return -ret; 263 } 264 #endif 265 266 if (max_nb_header_segs != 0) { 267 /* There is a limit on the number of header segments. */ 268 269 nb_header_segs += 270 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg, 271 &header_len_remaining); 272 273 if (unlikely(nb_header_segs > max_nb_header_segs)) { 274 /* 275 * The number of header segments is too large. 276 * 277 * If TSO is requested and if the datapath supports 278 * linearisation of TSO headers, allow the packet 279 * to proceed with additional checks below. 280 * Otherwise, throw an error. 281 */ 282 if ((m->ol_flags & PKT_TX_TCP_SEG) == 0 || 283 tso_bounce_buffer_len == 0) 284 return EINVAL; 285 } 286 } 287 288 if (m->ol_flags & PKT_TX_TCP_SEG) { 289 switch (m->ol_flags & PKT_TX_TUNNEL_MASK) { 290 case 0: 291 break; 292 case PKT_TX_TUNNEL_VXLAN: 293 /* FALLTHROUGH */ 294 case PKT_TX_TUNNEL_GENEVE: 295 if (!(m->ol_flags & 296 (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IPV6))) 297 return EINVAL; 298 } 299 300 if (unlikely(tcph_off > tso_tcp_header_offset_limit)) 301 return EINVAL; 302 303 descs_required += nb_tso_descs; 304 305 /* 306 * If headers segments are already counted above, here 307 * nothing is done since remaining length is smaller 308 * then current segment size. 309 */ 310 nb_header_segs += 311 sfc_dp_tx_pkt_extra_hdr_segs(&m_seg, 312 &header_len_remaining); 313 314 /* 315 * Extra descriptor which is required when (a part of) payload 316 * shares the same segment with (a part of) the header. 317 */ 318 if (rte_pktmbuf_data_len(m_seg) > header_len_remaining) 319 descs_required++; 320 321 if (tso_bounce_buffer_len != 0) { 322 if (nb_header_segs > 1 && 323 unlikely(header_len > tso_bounce_buffer_len)) { 324 /* 325 * Header linearization is required and 326 * the header is too big to be linearized 327 */ 328 return EINVAL; 329 } 330 } 331 } 332 333 /* 334 * The number of VLAN descriptors is added regardless of requested 335 * VLAN offload since VLAN is sticky and sending packet without VLAN 336 * insertion may require VLAN descriptor to reset the sticky to 0. 337 */ 338 descs_required += nb_vlan_descs; 339 340 /* 341 * Max fill level must be sufficient to hold all required descriptors 342 * to send the packet entirely. 343 */ 344 if (descs_required > max_fill_level) 345 return ENOBUFS; 346 347 return 0; 348 } 349 350 extern struct sfc_dp_tx sfc_efx_tx; 351 extern struct sfc_dp_tx sfc_ef10_tx; 352 extern struct sfc_dp_tx sfc_ef10_simple_tx; 353 extern struct sfc_dp_tx sfc_ef100_tx; 354 355 #ifdef __cplusplus 356 } 357 #endif 358 #endif /* _SFC_DP_TX_H */ 359