1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #ifndef __OCTEONTX_RXTX_H__ 6 #define __OCTEONTX_RXTX_H__ 7 8 #include <rte_ethdev_driver.h> 9 10 #define OFFLOAD_FLAGS \ 11 uint16_t rx_offload_flags; \ 12 uint16_t tx_offload_flags 13 14 #define BIT(nr) (1UL << (nr)) 15 16 #define OCCTX_RX_OFFLOAD_NONE (0) 17 #define OCCTX_RX_OFFLOAD_RSS_F BIT(0) 18 #define OCCTX_RX_MULTI_SEG_F BIT(15) 19 20 #define OCCTX_TX_OFFLOAD_NONE (0) 21 22 #define OCCTX_TX_MULTI_SEG_F BIT(15) 23 /* Packet type table */ 24 #define PTYPE_SIZE OCCTX_PKI_LTYPE_LAST 25 26 static const uint32_t __rte_cache_aligned 27 ptype_table[PTYPE_SIZE][PTYPE_SIZE][PTYPE_SIZE] = { 28 [LC_NONE][LE_NONE][LF_NONE] = RTE_PTYPE_UNKNOWN, 29 [LC_NONE][LE_NONE][LF_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 30 [LC_NONE][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L4_FRAG, 31 [LC_NONE][LE_NONE][LF_IPCOMP] = RTE_PTYPE_UNKNOWN, 32 [LC_NONE][LE_NONE][LF_TCP] = RTE_PTYPE_L4_TCP, 33 [LC_NONE][LE_NONE][LF_UDP] = RTE_PTYPE_L4_UDP, 34 [LC_NONE][LE_NONE][LF_GRE] = RTE_PTYPE_TUNNEL_GRE, 35 [LC_NONE][LE_NONE][LF_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 36 [LC_NONE][LE_NONE][LF_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 37 [LC_NONE][LE_NONE][LF_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 38 39 [LC_IPV4][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 40 [LC_IPV4][LE_NONE][LF_IPSEC_ESP] = 41 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4, 42 [LC_IPV4][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG, 43 [LC_IPV4][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 44 [LC_IPV4][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 45 [LC_IPV4][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 46 [LC_IPV4][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE, 47 [LC_IPV4][LE_NONE][LF_UDP_GENEVE] = 48 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE, 49 [LC_IPV4][LE_NONE][LF_UDP_VXLAN] = 50 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN, 51 [LC_IPV4][LE_NONE][LF_NVGRE] = 52 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 53 54 [LC_IPV4_OPT][LE_NONE][LF_NONE] = 55 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 56 [LC_IPV4_OPT][LE_NONE][LF_IPSEC_ESP] = 57 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L3_IPV4, 58 [LC_IPV4_OPT][LE_NONE][LF_IPFRAG] = 59 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG, 60 [LC_IPV4_OPT][LE_NONE][LF_IPCOMP] = 61 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 62 [LC_IPV4_OPT][LE_NONE][LF_TCP] = 63 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 64 [LC_IPV4_OPT][LE_NONE][LF_UDP] = 65 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 66 [LC_IPV4_OPT][LE_NONE][LF_GRE] = 67 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE, 68 [LC_IPV4_OPT][LE_NONE][LF_UDP_GENEVE] = 69 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GENEVE, 70 [LC_IPV4_OPT][LE_NONE][LF_UDP_VXLAN] = 71 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_VXLAN, 72 [LC_IPV4_OPT][LE_NONE][LF_NVGRE] = 73 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_NVGRE, 74 75 [LC_IPV6][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 76 [LC_IPV6][LE_NONE][LF_IPSEC_ESP] = 77 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4, 78 [LC_IPV6][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG, 79 [LC_IPV6][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 80 [LC_IPV6][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 81 [LC_IPV6][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 82 [LC_IPV6][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE, 83 [LC_IPV6][LE_NONE][LF_UDP_GENEVE] = 84 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE, 85 [LC_IPV6][LE_NONE][LF_UDP_VXLAN] = 86 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN, 87 [LC_IPV6][LE_NONE][LF_NVGRE] = 88 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 89 [LC_IPV6_OPT][LE_NONE][LF_NONE] = 90 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 91 [LC_IPV6_OPT][LE_NONE][LF_IPSEC_ESP] = 92 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L3_IPV4, 93 [LC_IPV6_OPT][LE_NONE][LF_IPFRAG] = 94 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG, 95 [LC_IPV6_OPT][LE_NONE][LF_IPCOMP] = 96 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 97 [LC_IPV6_OPT][LE_NONE][LF_TCP] = 98 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 99 [LC_IPV6_OPT][LE_NONE][LF_UDP] = 100 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 101 [LC_IPV6_OPT][LE_NONE][LF_GRE] = 102 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE, 103 [LC_IPV6_OPT][LE_NONE][LF_UDP_GENEVE] = 104 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GENEVE, 105 [LC_IPV6_OPT][LE_NONE][LF_UDP_VXLAN] = 106 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_VXLAN, 107 [LC_IPV6_OPT][LE_NONE][LF_NVGRE] = 108 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_NVGRE, 109 110 }; 111 112 113 static __rte_always_inline uint16_t 114 __octeontx_xmit_prepare(struct rte_mbuf *tx_pkt, uint64_t *cmd_buf, 115 const uint16_t flag __rte_unused) 116 { 117 uint16_t gaura_id, nb_desc = 0; 118 119 /* Setup PKO_SEND_HDR_S */ 120 cmd_buf[nb_desc++] = tx_pkt->data_len & 0xffff; 121 cmd_buf[nb_desc++] = 0x0; 122 123 /* Mark mempool object as "put" since it is freed by PKO */ 124 if (!(cmd_buf[0] & (1ULL << 58))) 125 __mempool_check_cookies(tx_pkt->pool, (void **)&tx_pkt, 126 1, 0); 127 /* Get the gaura Id */ 128 gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t) 129 tx_pkt->pool->pool_id); 130 131 /* Setup PKO_SEND_BUFLINK_S */ 132 cmd_buf[nb_desc++] = PKO_SEND_BUFLINK_SUBDC | 133 PKO_SEND_BUFLINK_LDTYPE(0x1ull) | 134 PKO_SEND_BUFLINK_GAUAR((long)gaura_id) | 135 tx_pkt->data_len; 136 cmd_buf[nb_desc++] = rte_mbuf_data_iova(tx_pkt); 137 138 return nb_desc; 139 } 140 141 static __rte_always_inline uint16_t 142 __octeontx_xmit_mseg_prepare(struct rte_mbuf *tx_pkt, uint64_t *cmd_buf, 143 const uint16_t flag __rte_unused) 144 { 145 uint16_t nb_segs, nb_desc = 0; 146 uint16_t gaura_id, len = 0; 147 struct rte_mbuf *m_next = NULL; 148 149 nb_segs = tx_pkt->nb_segs; 150 /* Setup PKO_SEND_HDR_S */ 151 cmd_buf[nb_desc++] = tx_pkt->pkt_len & 0xffff; 152 cmd_buf[nb_desc++] = 0x0; 153 154 do { 155 m_next = tx_pkt->next; 156 /* To handle case where mbufs belong to diff pools, like 157 * fragmentation 158 */ 159 gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t) 160 tx_pkt->pool->pool_id); 161 162 /* Setup PKO_SEND_GATHER_S */ 163 cmd_buf[nb_desc] = PKO_SEND_GATHER_SUBDC | 164 PKO_SEND_GATHER_LDTYPE(0x1ull) | 165 PKO_SEND_GATHER_GAUAR((long)gaura_id) | 166 tx_pkt->data_len; 167 168 /* Mark mempool object as "put" since it is freed by 169 * PKO. 170 */ 171 if (!(cmd_buf[nb_desc] & (1ULL << 57))) { 172 tx_pkt->next = NULL; 173 __mempool_check_cookies(tx_pkt->pool, 174 (void **)&tx_pkt, 1, 0); 175 } 176 nb_desc++; 177 178 cmd_buf[nb_desc++] = rte_mbuf_data_iova(tx_pkt); 179 180 nb_segs--; 181 len += tx_pkt->data_len; 182 tx_pkt = m_next; 183 } while (nb_segs); 184 185 return nb_desc; 186 } 187 188 static __rte_always_inline uint16_t 189 __octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 190 uint16_t nb_pkts, uint64_t *cmd_buf, 191 const uint16_t flags) 192 { 193 struct octeontx_txq *txq = tx_queue; 194 octeontx_dq_t *dq = &txq->dq; 195 uint16_t count = 0, nb_desc; 196 rte_cio_wmb(); 197 198 while (count < nb_pkts) { 199 if (unlikely(*((volatile int64_t *)dq->fc_status_va) < 0)) 200 break; 201 202 if (flags & OCCTX_TX_MULTI_SEG_F) { 203 nb_desc = __octeontx_xmit_mseg_prepare(tx_pkts[count], 204 cmd_buf, flags); 205 } else { 206 nb_desc = __octeontx_xmit_prepare(tx_pkts[count], 207 cmd_buf, flags); 208 } 209 210 octeontx_reg_lmtst(dq->lmtline_va, dq->ioreg_va, cmd_buf, 211 nb_desc); 212 213 count++; 214 } 215 return count; 216 } 217 218 uint16_t 219 octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 220 221 #define MULT_F OCCTX_TX_MULTI_SEG_F 222 /* [NOFF] [MULTI_SEG] */ 223 #define OCCTX_TX_FASTPATH_MODES \ 224 T(no_offload, 0, 4, OCCTX_TX_OFFLOAD_NONE) \ 225 T(mseg, 1, 14, MULT_F) \ 226 227 #endif /* __OCTEONTX_RXTX_H__ */ 228