1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #ifndef _NGBE_RXTX_H_ 7 #define _NGBE_RXTX_H_ 8 9 /***************************************************************************** 10 * Receive Descriptor 11 *****************************************************************************/ 12 struct ngbe_rx_desc { 13 struct { 14 union { 15 rte_le32_t dw0; 16 struct { 17 rte_le16_t pkt; 18 rte_le16_t hdr; 19 } lo; 20 }; 21 union { 22 rte_le32_t dw1; 23 struct { 24 rte_le16_t ipid; 25 rte_le16_t csum; 26 } hi; 27 }; 28 } qw0; /* also as r.pkt_addr */ 29 struct { 30 union { 31 rte_le32_t dw2; 32 struct { 33 rte_le32_t status; 34 } lo; 35 }; 36 union { 37 rte_le32_t dw3; 38 struct { 39 rte_le16_t len; 40 rte_le16_t tag; 41 } hi; 42 }; 43 } qw1; /* also as r.hdr_addr */ 44 }; 45 46 /* @ngbe_rx_desc.qw0 */ 47 #define NGBE_RXD_PKTADDR(rxd, v) \ 48 (((volatile __le64 *)(rxd))[0] = cpu_to_le64(v)) 49 50 /* @ngbe_rx_desc.qw1 */ 51 #define NGBE_RXD_HDRADDR(rxd, v) \ 52 (((volatile __le64 *)(rxd))[1] = cpu_to_le64(v)) 53 54 /* @ngbe_rx_desc.dw0 */ 55 #define NGBE_RXD_RSSTYPE(dw) RS(dw, 0, 0xF) 56 #define NGBE_RSSTYPE_NONE 0 57 #define NGBE_RSSTYPE_IPV4TCP 1 58 #define NGBE_RSSTYPE_IPV4 2 59 #define NGBE_RSSTYPE_IPV6TCP 3 60 #define NGBE_RSSTYPE_IPV4SCTP 4 61 #define NGBE_RSSTYPE_IPV6 5 62 #define NGBE_RSSTYPE_IPV6SCTP 6 63 #define NGBE_RSSTYPE_IPV4UDP 7 64 #define NGBE_RSSTYPE_IPV6UDP 8 65 #define NGBE_RSSTYPE_FDIR 15 66 #define NGBE_RXD_SECTYPE(dw) RS(dw, 4, 0x3) 67 #define NGBE_RXD_SECTYPE_NONE LS(0, 4, 0x3) 68 #define NGBE_RXD_SECTYPE_IPSECESP LS(2, 4, 0x3) 69 #define NGBE_RXD_SECTYPE_IPSECAH LS(3, 4, 0x3) 70 #define NGBE_RXD_TPIDSEL(dw) RS(dw, 6, 0x7) 71 #define NGBE_RXD_PTID(dw) RS(dw, 9, 0xFF) 72 #define NGBE_RXD_RSCCNT(dw) RS(dw, 17, 0xF) 73 #define NGBE_RXD_HDRLEN(dw) RS(dw, 21, 0x3FF) 74 #define NGBE_RXD_SPH MS(31, 0x1) 75 76 /* @ngbe_rx_desc.dw1 */ 77 /** bit 0-31, as rss hash when **/ 78 #define NGBE_RXD_RSSHASH(rxd) ((rxd)->qw0.dw1) 79 80 /** bit 0-31, as ip csum when **/ 81 #define NGBE_RXD_IPID(rxd) ((rxd)->qw0.hi.ipid) 82 #define NGBE_RXD_CSUM(rxd) ((rxd)->qw0.hi.csum) 83 84 /* @ngbe_rx_desc.dw2 */ 85 #define NGBE_RXD_STATUS(rxd) ((rxd)->qw1.lo.status) 86 /** bit 0-1 **/ 87 #define NGBE_RXD_STAT_DD MS(0, 0x1) /* Descriptor Done */ 88 #define NGBE_RXD_STAT_EOP MS(1, 0x1) /* End of Packet */ 89 /** bit 2-31, when EOP=0 **/ 90 #define NGBE_RXD_NEXTP_RESV(v) LS(v, 2, 0x3) 91 #define NGBE_RXD_NEXTP(dw) RS(dw, 4, 0xFFFF) /* Next Descriptor */ 92 /** bit 2-31, when EOP=1 **/ 93 #define NGBE_RXD_PKT_CLS_MASK MS(2, 0x7) /* Packet Class */ 94 #define NGBE_RXD_PKT_CLS_TC_RSS LS(0, 2, 0x7) /* RSS Hash */ 95 #define NGBE_RXD_PKT_CLS_FLM LS(1, 2, 0x7) /* FDir Match */ 96 #define NGBE_RXD_PKT_CLS_SYN LS(2, 2, 0x7) /* TCP Sync */ 97 #define NGBE_RXD_PKT_CLS_5TUPLE LS(3, 2, 0x7) /* 5 Tuple */ 98 #define NGBE_RXD_PKT_CLS_ETF LS(4, 2, 0x7) /* Ethertype Filter */ 99 #define NGBE_RXD_STAT_VLAN MS(5, 0x1) /* IEEE VLAN Packet */ 100 #define NGBE_RXD_STAT_UDPCS MS(6, 0x1) /* UDP xsum calculated */ 101 #define NGBE_RXD_STAT_L4CS MS(7, 0x1) /* L4 xsum calculated */ 102 #define NGBE_RXD_STAT_IPCS MS(8, 0x1) /* IP xsum calculated */ 103 #define NGBE_RXD_STAT_PIF MS(9, 0x1) /* Non-unicast address */ 104 #define NGBE_RXD_STAT_EIPCS MS(10, 0x1) /* Encap IP xsum calculated */ 105 #define NGBE_RXD_STAT_VEXT MS(11, 0x1) /* Multi-VLAN */ 106 #define NGBE_RXD_STAT_IPV6EX MS(12, 0x1) /* IPv6 with option header */ 107 #define NGBE_RXD_STAT_LLINT MS(13, 0x1) /* Pkt caused LLI */ 108 #define NGBE_RXD_STAT_1588 MS(14, 0x1) /* IEEE1588 Time Stamp */ 109 #define NGBE_RXD_STAT_SECP MS(15, 0x1) /* Security Processing */ 110 #define NGBE_RXD_STAT_LB MS(16, 0x1) /* Loopback Status */ 111 /*** bit 17-30, when PTYPE=IP ***/ 112 #define NGBE_RXD_STAT_BMC MS(17, 0x1) /* PTYPE=IP, BMC status */ 113 #define NGBE_RXD_ERR_HBO MS(23, 0x1) /* Header Buffer Overflow */ 114 #define NGBE_RXD_ERR_EIPCS MS(26, 0x1) /* Encap IP header error */ 115 #define NGBE_RXD_ERR_SECERR MS(27, 0x1) /* macsec or ipsec error */ 116 #define NGBE_RXD_ERR_RXE MS(29, 0x1) /* Any MAC Error */ 117 #define NGBE_RXD_ERR_L4CS MS(30, 0x1) /* TCP/UDP xsum error */ 118 #define NGBE_RXD_ERR_IPCS MS(31, 0x1) /* IP xsum error */ 119 #define NGBE_RXD_ERR_CSUM(dw) RS(dw, 30, 0x3) 120 121 /* @ngbe_rx_desc.dw3 */ 122 #define NGBE_RXD_LENGTH(rxd) ((rxd)->qw1.hi.len) 123 #define NGBE_RXD_VLAN(rxd) ((rxd)->qw1.hi.tag) 124 125 /***************************************************************************** 126 * Transmit Descriptor 127 *****************************************************************************/ 128 /** 129 * Transmit Context Descriptor (NGBE_TXD_TYP=CTXT) 130 **/ 131 struct ngbe_tx_ctx_desc { 132 rte_le32_t dw0; /* w.vlan_macip_lens */ 133 rte_le32_t dw1; /* w.seqnum_seed */ 134 rte_le32_t dw2; /* w.type_tucmd_mlhl */ 135 rte_le32_t dw3; /* w.mss_l4len_idx */ 136 }; 137 138 /* @ngbe_tx_ctx_desc.dw0 */ 139 #define NGBE_TXD_IPLEN(v) LS(v, 0, 0x1FF) /* ip/fcoe header end */ 140 #define NGBE_TXD_MACLEN(v) LS(v, 9, 0x7F) /* desc mac len */ 141 #define NGBE_TXD_VLAN(v) LS(v, 16, 0xFFFF) /* vlan tag */ 142 143 /* @ngbe_tx_ctx_desc.dw1 */ 144 /*** bit 0-31, when NGBE_TXD_DTYP_FCOE=0 ***/ 145 #define NGBE_TXD_IPSEC_SAIDX(v) LS(v, 0, 0x3FF) /* ipsec SA index */ 146 #define NGBE_TXD_ETYPE(v) LS(v, 11, 0x1) /* tunnel type */ 147 #define NGBE_TXD_ETYPE_UDP LS(0, 11, 0x1) 148 #define NGBE_TXD_ETYPE_GRE LS(1, 11, 0x1) 149 #define NGBE_TXD_EIPLEN(v) LS(v, 12, 0x7F) /* tunnel ip header */ 150 #define NGBE_TXD_DTYP_FCOE MS(16, 0x1) /* FCoE/IP descriptor */ 151 #define NGBE_TXD_ETUNLEN(v) LS(v, 21, 0xFF) /* tunnel header */ 152 #define NGBE_TXD_DECTTL(v) LS(v, 29, 0xF) /* decrease ip TTL */ 153 154 /* @ngbe_tx_ctx_desc.dw2 */ 155 #define NGBE_TXD_IPSEC_ESPLEN(v) LS(v, 1, 0x1FF) /* ipsec ESP length */ 156 #define NGBE_TXD_SNAP MS(10, 0x1) /* SNAP indication */ 157 #define NGBE_TXD_TPID_SEL(v) LS(v, 11, 0x7) /* vlan tag index */ 158 #define NGBE_TXD_IPSEC_ESP MS(14, 0x1) /* ipsec type: esp=1 ah=0 */ 159 #define NGBE_TXD_IPSEC_ESPENC MS(15, 0x1) /* ESP encrypt */ 160 #define NGBE_TXD_CTXT MS(20, 0x1) /* context descriptor */ 161 #define NGBE_TXD_PTID(v) LS(v, 24, 0xFF) /* packet type */ 162 /* @ngbe_tx_ctx_desc.dw3 */ 163 #define NGBE_TXD_DD MS(0, 0x1) /* descriptor done */ 164 #define NGBE_TXD_IDX(v) LS(v, 4, 0x1) /* ctxt desc index */ 165 #define NGBE_TXD_L4LEN(v) LS(v, 8, 0xFF) /* l4 header length */ 166 #define NGBE_TXD_MSS(v) LS(v, 16, 0xFFFF) /* l4 MSS */ 167 168 /** 169 * Transmit Data Descriptor (NGBE_TXD_TYP=DATA) 170 **/ 171 struct ngbe_tx_desc { 172 rte_le64_t qw0; /* r.buffer_addr , w.reserved */ 173 rte_le32_t dw2; /* r.cmd_type_len, w.nxtseq_seed */ 174 rte_le32_t dw3; /* r.olinfo_status, w.status */ 175 }; 176 177 /* @ngbe_tx_desc.dw2 */ 178 #define NGBE_TXD_DATLEN(v) ((0xFFFF & (v))) /* data buffer length */ 179 #define NGBE_TXD_1588 ((0x1) << 19) /* IEEE1588 time stamp */ 180 #define NGBE_TXD_DATA ((0x0) << 20) /* data descriptor */ 181 #define NGBE_TXD_EOP ((0x1) << 24) /* End of Packet */ 182 #define NGBE_TXD_FCS ((0x1) << 25) /* Insert FCS */ 183 #define NGBE_TXD_LINKSEC ((0x1) << 26) /* Insert LinkSec */ 184 #define NGBE_TXD_ECU ((0x1) << 28) /* forward to ECU */ 185 #define NGBE_TXD_CNTAG ((0x1) << 29) /* insert CN tag */ 186 #define NGBE_TXD_VLE ((0x1) << 30) /* insert VLAN tag */ 187 #define NGBE_TXD_TSE ((0x1) << 31) /* transmit segmentation */ 188 189 #define NGBE_TXD_FLAGS (NGBE_TXD_FCS | NGBE_TXD_EOP) 190 191 /* @ngbe_tx_desc.dw3 */ 192 #define NGBE_TXD_DD_UNUSED NGBE_TXD_DD 193 #define NGBE_TXD_IDX_UNUSED(v) NGBE_TXD_IDX(v) 194 #define NGBE_TXD_CC ((0x1) << 7) /* check context */ 195 #define NGBE_TXD_IPSEC ((0x1) << 8) /* request ipsec offload */ 196 #define NGBE_TXD_L4CS ((0x1) << 9) /* insert TCP/UDP/SCTP csum */ 197 #define NGBE_TXD_IPCS ((0x1) << 10) /* insert IPv4 csum */ 198 #define NGBE_TXD_EIPCS ((0x1) << 11) /* insert outer IP csum */ 199 #define NGBE_TXD_MNGFLT ((0x1) << 12) /* enable management filter */ 200 #define NGBE_TXD_PAYLEN(v) ((0x7FFFF & (v)) << 13) /* payload length */ 201 202 #define RTE_PMD_NGBE_TX_MAX_BURST 32 203 #define RTE_PMD_NGBE_RX_MAX_BURST 32 204 #define RTE_NGBE_TX_MAX_FREE_BUF_SZ 64 205 206 #define RTE_NGBE_DESCS_PER_LOOP 4 207 208 #define RX_RING_SZ ((NGBE_RING_DESC_MAX + RTE_PMD_NGBE_RX_MAX_BURST) * \ 209 sizeof(struct ngbe_rx_desc)) 210 211 #define rte_packet_prefetch(p) rte_prefetch1(p) 212 213 #define RTE_NGBE_REGISTER_POLL_WAIT_10_MS 10 214 #define RTE_NGBE_WAIT_100_US 100 215 216 #define NGBE_TX_MAX_SEG 40 217 218 #ifndef DEFAULT_TX_FREE_THRESH 219 #define DEFAULT_TX_FREE_THRESH 32 220 #endif 221 222 /** 223 * Structure associated with each descriptor of the Rx ring of a Rx queue. 224 */ 225 struct ngbe_rx_entry { 226 struct rte_mbuf *mbuf; /**< mbuf associated with Rx descriptor. */ 227 }; 228 229 struct ngbe_scattered_rx_entry { 230 struct rte_mbuf *fbuf; /**< First segment of the fragmented packet. */ 231 }; 232 233 /** 234 * Structure associated with each descriptor of the Tx ring of a Tx queue. 235 */ 236 struct ngbe_tx_entry { 237 struct rte_mbuf *mbuf; /**< mbuf associated with Tx desc, if any. */ 238 uint16_t next_id; /**< Index of next descriptor in ring. */ 239 uint16_t last_id; /**< Index of last scattered descriptor. */ 240 }; 241 242 /** 243 * Structure associated with each descriptor of the Tx ring of a Tx queue. 244 */ 245 struct ngbe_tx_entry_v { 246 struct rte_mbuf *mbuf; /**< mbuf associated with Tx desc, if any. */ 247 }; 248 249 /** 250 * Structure associated with each Rx queue. 251 */ 252 struct ngbe_rx_queue { 253 struct rte_mempool *mb_pool; /**< mbuf pool to populate Rx ring */ 254 uint64_t rx_ring_phys_addr; /**< Rx ring DMA address */ 255 volatile uint32_t *rdt_reg_addr; /**< RDT register address */ 256 volatile uint32_t *rdh_reg_addr; /**< RDH register address */ 257 258 volatile struct ngbe_rx_desc *rx_ring; /**< Rx ring virtual address */ 259 /** address of Rx software ring */ 260 struct ngbe_rx_entry *sw_ring; 261 /** address of scattered Rx software ring */ 262 struct ngbe_scattered_rx_entry *sw_sc_ring; 263 264 struct rte_mbuf *pkt_first_seg; /**< First segment of current packet */ 265 struct rte_mbuf *pkt_last_seg; /**< Last segment of current packet */ 266 uint64_t mbuf_initializer; /**< value to init mbufs */ 267 uint16_t nb_rx_desc; /**< number of Rx descriptors */ 268 uint16_t rx_tail; /**< current value of RDT register */ 269 uint16_t nb_rx_hold; /**< number of held free Rx desc */ 270 271 uint16_t rx_nb_avail; /**< nr of staged pkts ready to ret to app */ 272 uint16_t rx_next_avail; /**< idx of next staged pkt to ret to app */ 273 uint16_t rx_free_trigger; /**< triggers rx buffer allocation */ 274 275 uint8_t rx_using_sse; /**< indicates that vector Rx is in use */ 276 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM) 277 uint16_t rxrearm_nb; /**< number of remaining to be re-armed */ 278 uint16_t rxrearm_start; /**< the idx we start the re-arming from */ 279 #endif 280 uint16_t rx_free_thresh; /**< max free Rx desc to hold */ 281 uint16_t queue_id; /**< RX queue index */ 282 uint16_t reg_idx; /**< RX queue register index */ 283 uint16_t port_id; /**< Device port identifier */ 284 uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ 285 uint8_t drop_en; /**< If not 0, set SRRCTL.Drop_En */ 286 uint8_t rx_deferred_start; /**< not in global dev start */ 287 /** flags to set in mbuf when a vlan is detected */ 288 uint64_t vlan_flags; 289 uint64_t offloads; /**< Rx offloads with RTE_ETH_RX_OFFLOAD_* */ 290 /** need to alloc dummy mbuf, for wraparound when scanning hw ring */ 291 struct rte_mbuf fake_mbuf; 292 /** hold packets to return to application */ 293 struct rte_mbuf *rx_stage[RTE_PMD_NGBE_RX_MAX_BURST * 2]; 294 const struct rte_memzone *mz; 295 }; 296 297 /** 298 * NGBE CTX Constants 299 */ 300 enum ngbe_ctx_num { 301 NGBE_CTX_0 = 0, /**< CTX0 */ 302 NGBE_CTX_1 = 1, /**< CTX1 */ 303 NGBE_CTX_NUM = 2, /**< CTX NUMBER */ 304 }; 305 306 /** Offload features */ 307 union ngbe_tx_offload { 308 uint64_t data[2]; 309 struct { 310 uint64_t ptid:8; /**< Packet Type Identifier. */ 311 uint64_t l2_len:7; /**< L2 (MAC) Header Length. */ 312 uint64_t l3_len:9; /**< L3 (IP) Header Length. */ 313 uint64_t l4_len:8; /**< L4 (TCP/UDP) Header Length. */ 314 uint64_t tso_segsz:16; /**< TCP TSO segment size */ 315 uint64_t vlan_tci:16; 316 /**< VLAN Tag Control Identifier (CPU order). */ 317 318 /* fields for TX offloading of tunnels */ 319 uint64_t outer_tun_len:8; /**< Outer TUN (Tunnel) Hdr Length. */ 320 uint64_t outer_l2_len:8; /**< Outer L2 (MAC) Hdr Length. */ 321 uint64_t outer_l3_len:16; /**< Outer L3 (IP) Hdr Length. */ 322 }; 323 }; 324 325 /** 326 * Structure to check if new context need be built 327 */ 328 struct ngbe_ctx_info { 329 uint64_t flags; /**< ol_flags for context build. */ 330 /**< tx offload: vlan, tso, l2-l3-l4 lengths. */ 331 union ngbe_tx_offload tx_offload; 332 /** compare mask for tx offload. */ 333 union ngbe_tx_offload tx_offload_mask; 334 }; 335 336 /** 337 * Structure associated with each Tx queue. 338 */ 339 struct ngbe_tx_queue { 340 /** Tx ring virtual address */ 341 volatile struct ngbe_tx_desc *tx_ring; 342 343 uint64_t tx_ring_phys_addr; /**< Tx ring DMA address */ 344 union { 345 /**< address of SW ring for scalar PMD. */ 346 struct ngbe_tx_entry *sw_ring; 347 /**< address of SW ring for vector PMD */ 348 struct ngbe_tx_entry_v *sw_ring_v; 349 }; 350 volatile uint32_t *tdt_reg_addr; /**< Address of TDT register */ 351 volatile uint32_t *tdc_reg_addr; /**< Address of TDC register */ 352 uint16_t nb_tx_desc; /**< number of Tx descriptors */ 353 uint16_t tx_tail; /**< current value of TDT reg */ 354 /** 355 * Start freeing Tx buffers if there are less free descriptors than 356 * this value. 357 */ 358 uint16_t tx_free_thresh; 359 /** Index to last Tx descriptor to have been cleaned */ 360 uint16_t last_desc_cleaned; 361 /** Total number of Tx descriptors ready to be allocated */ 362 uint16_t nb_tx_free; 363 uint16_t tx_next_dd; /**< next desc to scan for DD bit */ 364 uint16_t queue_id; /**< Tx queue index */ 365 uint16_t reg_idx; /**< Tx queue register index */ 366 uint16_t port_id; /**< Device port identifier */ 367 uint8_t pthresh; /**< Prefetch threshold register */ 368 uint8_t hthresh; /**< Host threshold register */ 369 uint8_t wthresh; /**< Write-back threshold reg */ 370 uint64_t offloads; /**< Tx offload flags */ 371 uint32_t ctx_curr; /**< Hardware context states */ 372 /** Hardware context0 history */ 373 struct ngbe_ctx_info ctx_cache[NGBE_CTX_NUM]; 374 uint8_t tx_deferred_start; /**< not in global dev start */ 375 376 const struct ngbe_txq_ops *ops; /**< txq ops */ 377 const struct rte_memzone *mz; 378 uint64_t desc_error; 379 }; 380 381 struct ngbe_txq_ops { 382 void (*release_mbufs)(struct ngbe_tx_queue *txq); 383 void (*free_swring)(struct ngbe_tx_queue *txq); 384 void (*reset)(struct ngbe_tx_queue *txq); 385 }; 386 387 /* Takes an ethdev and a queue and sets up the tx function to be used based on 388 * the queue parameters. Used in tx_queue_setup by primary process and then 389 * in dev_init by secondary process when attaching to an existing ethdev. 390 */ 391 void ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq); 392 393 void ngbe_set_rx_function(struct rte_eth_dev *dev); 394 uint16_t ngbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, 395 uint16_t nb_pkts); 396 uint16_t ngbe_recv_scattered_pkts_vec(void *rx_queue, 397 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 398 int ngbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev); 399 int ngbe_rxq_vec_setup(struct ngbe_rx_queue *rxq); 400 void ngbe_rx_queue_release_mbufs_vec(struct ngbe_rx_queue *rxq); 401 uint16_t ngbe_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 402 uint16_t nb_pkts); 403 int ngbe_txq_vec_setup(struct ngbe_tx_queue *txq); 404 int ngbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt); 405 406 uint64_t ngbe_get_tx_port_offloads(struct rte_eth_dev *dev); 407 uint64_t ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev); 408 uint64_t ngbe_get_rx_port_offloads(struct rte_eth_dev *dev); 409 410 #endif /* _NGBE_RXTX_H_ */ 411