1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #ifndef __OCTEONTX_RXTX_H__ 6 #define __OCTEONTX_RXTX_H__ 7 8 #include <rte_ethdev_driver.h> 9 10 #define OFFLOAD_FLAGS \ 11 uint16_t rx_offload_flags; \ 12 uint16_t tx_offload_flags 13 14 #define BIT(nr) (1UL << (nr)) 15 16 #define OCCTX_RX_OFFLOAD_NONE (0) 17 #define OCCTX_RX_OFFLOAD_RSS_F BIT(0) 18 #define OCCTX_RX_MULTI_SEG_F BIT(15) 19 20 #define OCCTX_TX_OFFLOAD_NONE (0) 21 #define OCCTX_TX_OFFLOAD_MBUF_NOFF_F BIT(3) 22 23 #define OCCTX_TX_MULTI_SEG_F BIT(15) 24 /* Packet type table */ 25 #define PTYPE_SIZE OCCTX_PKI_LTYPE_LAST 26 27 static const uint32_t __rte_cache_aligned 28 ptype_table[PTYPE_SIZE][PTYPE_SIZE][PTYPE_SIZE] = { 29 [LC_NONE][LE_NONE][LF_NONE] = RTE_PTYPE_UNKNOWN, 30 [LC_NONE][LE_NONE][LF_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 31 [LC_NONE][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L4_FRAG, 32 [LC_NONE][LE_NONE][LF_IPCOMP] = RTE_PTYPE_UNKNOWN, 33 [LC_NONE][LE_NONE][LF_TCP] = RTE_PTYPE_L4_TCP, 34 [LC_NONE][LE_NONE][LF_UDP] = RTE_PTYPE_L4_UDP, 35 [LC_NONE][LE_NONE][LF_GRE] = RTE_PTYPE_TUNNEL_GRE, 36 [LC_NONE][LE_NONE][LF_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 37 [LC_NONE][LE_NONE][LF_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 38 [LC_NONE][LE_NONE][LF_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 39 40 [LC_IPV4][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 41 [LC_IPV4][LE_NONE][LF_IPSEC_ESP] = 42 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4, 43 [LC_IPV4][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG, 44 [LC_IPV4][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 45 [LC_IPV4][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 46 [LC_IPV4][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 47 [LC_IPV4][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE, 48 [LC_IPV4][LE_NONE][LF_UDP_GENEVE] = 49 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE, 50 [LC_IPV4][LE_NONE][LF_UDP_VXLAN] = 51 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN, 52 [LC_IPV4][LE_NONE][LF_NVGRE] = 53 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 54 55 [LC_IPV4_OPT][LE_NONE][LF_NONE] = 56 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 57 [LC_IPV4_OPT][LE_NONE][LF_IPSEC_ESP] = 58 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L3_IPV4, 59 [LC_IPV4_OPT][LE_NONE][LF_IPFRAG] = 60 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG, 61 [LC_IPV4_OPT][LE_NONE][LF_IPCOMP] = 62 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 63 [LC_IPV4_OPT][LE_NONE][LF_TCP] = 64 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 65 [LC_IPV4_OPT][LE_NONE][LF_UDP] = 66 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 67 [LC_IPV4_OPT][LE_NONE][LF_GRE] = 68 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE, 69 [LC_IPV4_OPT][LE_NONE][LF_UDP_GENEVE] = 70 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GENEVE, 71 [LC_IPV4_OPT][LE_NONE][LF_UDP_VXLAN] = 72 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_VXLAN, 73 [LC_IPV4_OPT][LE_NONE][LF_NVGRE] = 74 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_NVGRE, 75 76 [LC_IPV6][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 77 [LC_IPV6][LE_NONE][LF_IPSEC_ESP] = 78 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4, 79 [LC_IPV6][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG, 80 [LC_IPV6][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 81 [LC_IPV6][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 82 [LC_IPV6][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 83 [LC_IPV6][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE, 84 [LC_IPV6][LE_NONE][LF_UDP_GENEVE] = 85 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE, 86 [LC_IPV6][LE_NONE][LF_UDP_VXLAN] = 87 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN, 88 [LC_IPV6][LE_NONE][LF_NVGRE] = 89 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 90 [LC_IPV6_OPT][LE_NONE][LF_NONE] = 91 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 92 [LC_IPV6_OPT][LE_NONE][LF_IPSEC_ESP] = 93 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L3_IPV4, 94 [LC_IPV6_OPT][LE_NONE][LF_IPFRAG] = 95 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG, 96 [LC_IPV6_OPT][LE_NONE][LF_IPCOMP] = 97 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 98 [LC_IPV6_OPT][LE_NONE][LF_TCP] = 99 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 100 [LC_IPV6_OPT][LE_NONE][LF_UDP] = 101 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 102 [LC_IPV6_OPT][LE_NONE][LF_GRE] = 103 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE, 104 [LC_IPV6_OPT][LE_NONE][LF_UDP_GENEVE] = 105 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GENEVE, 106 [LC_IPV6_OPT][LE_NONE][LF_UDP_VXLAN] = 107 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_VXLAN, 108 [LC_IPV6_OPT][LE_NONE][LF_NVGRE] = 109 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_NVGRE, 110 111 }; 112 113 114 static __rte_always_inline uint64_t 115 octeontx_pktmbuf_detach(struct rte_mbuf *m) 116 { 117 struct rte_mempool *mp = m->pool; 118 uint32_t mbuf_size, buf_len; 119 struct rte_mbuf *md; 120 uint16_t priv_size; 121 uint16_t refcount; 122 123 /* Update refcount of direct mbuf */ 124 md = rte_mbuf_from_indirect(m); 125 refcount = rte_mbuf_refcnt_update(md, -1); 126 127 priv_size = rte_pktmbuf_priv_size(mp); 128 mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size); 129 buf_len = rte_pktmbuf_data_room_size(mp); 130 131 m->priv_size = priv_size; 132 m->buf_addr = (char *)m + mbuf_size; 133 m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; 134 m->buf_len = (uint16_t)buf_len; 135 rte_pktmbuf_reset_headroom(m); 136 m->data_len = 0; 137 m->ol_flags = 0; 138 m->next = NULL; 139 m->nb_segs = 1; 140 141 /* Now indirect mbuf is safe to free */ 142 rte_pktmbuf_free(m); 143 144 if (refcount == 0) { 145 rte_mbuf_refcnt_set(md, 1); 146 md->data_len = 0; 147 md->ol_flags = 0; 148 md->next = NULL; 149 md->nb_segs = 1; 150 return 0; 151 } else { 152 return 1; 153 } 154 } 155 156 static __rte_always_inline uint64_t 157 octeontx_prefree_seg(struct rte_mbuf *m) 158 { 159 if (likely(rte_mbuf_refcnt_read(m) == 1)) { 160 if (!RTE_MBUF_DIRECT(m)) 161 return octeontx_pktmbuf_detach(m); 162 163 m->next = NULL; 164 m->nb_segs = 1; 165 return 0; 166 } else if (rte_mbuf_refcnt_update(m, -1) == 0) { 167 if (!RTE_MBUF_DIRECT(m)) 168 return octeontx_pktmbuf_detach(m); 169 170 rte_mbuf_refcnt_set(m, 1); 171 m->next = NULL; 172 m->nb_segs = 1; 173 return 0; 174 } 175 176 /* Mbuf is having refcount more than 1 so need not to be freed */ 177 return 1; 178 } 179 180 static __rte_always_inline uint16_t 181 __octeontx_xmit_prepare(struct rte_mbuf *tx_pkt, uint64_t *cmd_buf, 182 const uint16_t flag) 183 { 184 uint16_t gaura_id, nb_desc = 0; 185 186 /* Setup PKO_SEND_HDR_S */ 187 cmd_buf[nb_desc++] = tx_pkt->data_len & 0xffff; 188 cmd_buf[nb_desc++] = 0x0; 189 190 /* SEND_HDR[DF] bit controls if buffer is to be freed or 191 * not, as SG_DESC[I] and SEND_HDR[II] are clear. 192 */ 193 if (flag & OCCTX_TX_OFFLOAD_MBUF_NOFF_F) 194 cmd_buf[0] |= (octeontx_prefree_seg(tx_pkt) << 195 58); 196 197 /* Mark mempool object as "put" since it is freed by PKO */ 198 if (!(cmd_buf[0] & (1ULL << 58))) 199 __mempool_check_cookies(tx_pkt->pool, (void **)&tx_pkt, 200 1, 0); 201 /* Get the gaura Id */ 202 gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t) 203 tx_pkt->pool->pool_id); 204 205 /* Setup PKO_SEND_BUFLINK_S */ 206 cmd_buf[nb_desc++] = PKO_SEND_BUFLINK_SUBDC | 207 PKO_SEND_BUFLINK_LDTYPE(0x1ull) | 208 PKO_SEND_BUFLINK_GAUAR((long)gaura_id) | 209 tx_pkt->data_len; 210 cmd_buf[nb_desc++] = rte_mbuf_data_iova(tx_pkt); 211 212 return nb_desc; 213 } 214 215 static __rte_always_inline uint16_t 216 __octeontx_xmit_mseg_prepare(struct rte_mbuf *tx_pkt, uint64_t *cmd_buf, 217 const uint16_t flag) 218 { 219 uint16_t nb_segs, nb_desc = 0; 220 uint16_t gaura_id, len = 0; 221 struct rte_mbuf *m_next = NULL; 222 223 nb_segs = tx_pkt->nb_segs; 224 /* Setup PKO_SEND_HDR_S */ 225 cmd_buf[nb_desc++] = tx_pkt->pkt_len & 0xffff; 226 cmd_buf[nb_desc++] = 0x0; 227 228 do { 229 m_next = tx_pkt->next; 230 /* To handle case where mbufs belong to diff pools, like 231 * fragmentation 232 */ 233 gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t) 234 tx_pkt->pool->pool_id); 235 236 /* Setup PKO_SEND_GATHER_S */ 237 cmd_buf[nb_desc] = PKO_SEND_GATHER_SUBDC | 238 PKO_SEND_GATHER_LDTYPE(0x1ull) | 239 PKO_SEND_GATHER_GAUAR((long)gaura_id) | 240 tx_pkt->data_len; 241 242 /* SG_DESC[I] bit controls if buffer is to be freed or 243 * not, as SEND_HDR[DF] and SEND_HDR[II] are clear. 244 */ 245 if (flag & OCCTX_TX_OFFLOAD_MBUF_NOFF_F) { 246 cmd_buf[nb_desc] |= 247 (octeontx_prefree_seg(tx_pkt) << 57); 248 } 249 250 /* Mark mempool object as "put" since it is freed by 251 * PKO. 252 */ 253 if (!(cmd_buf[nb_desc] & (1ULL << 57))) { 254 tx_pkt->next = NULL; 255 __mempool_check_cookies(tx_pkt->pool, 256 (void **)&tx_pkt, 1, 0); 257 } 258 nb_desc++; 259 260 cmd_buf[nb_desc++] = rte_mbuf_data_iova(tx_pkt); 261 262 nb_segs--; 263 len += tx_pkt->data_len; 264 tx_pkt = m_next; 265 } while (nb_segs); 266 267 return nb_desc; 268 } 269 270 static __rte_always_inline uint16_t 271 __octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 272 uint16_t nb_pkts, uint64_t *cmd_buf, 273 const uint16_t flags) 274 { 275 struct octeontx_txq *txq = tx_queue; 276 octeontx_dq_t *dq = &txq->dq; 277 uint16_t count = 0, nb_desc; 278 rte_cio_wmb(); 279 280 while (count < nb_pkts) { 281 if (unlikely(*((volatile int64_t *)dq->fc_status_va) < 0)) 282 break; 283 284 if (flags & OCCTX_TX_MULTI_SEG_F) { 285 nb_desc = __octeontx_xmit_mseg_prepare(tx_pkts[count], 286 cmd_buf, flags); 287 } else { 288 nb_desc = __octeontx_xmit_prepare(tx_pkts[count], 289 cmd_buf, flags); 290 } 291 292 octeontx_reg_lmtst(dq->lmtline_va, dq->ioreg_va, cmd_buf, 293 nb_desc); 294 295 count++; 296 } 297 return count; 298 } 299 300 uint16_t 301 octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 302 303 #define NOFF_F OCCTX_TX_OFFLOAD_MBUF_NOFF_F 304 #define MULT_F OCCTX_TX_MULTI_SEG_F 305 /* [NOFF] [MULTI_SEG] */ 306 #define OCCTX_TX_FASTPATH_MODES \ 307 T(no_offload, 0, 0, 4, OCCTX_TX_OFFLOAD_NONE) \ 308 T(mseg, 0, 1, 14, MULT_F) \ 309 T(noff, 1, 0, 4, NOFF_F) \ 310 T(noff_mseg, 1, 1, 14, NOFF_F | MULT_F) 311 312 #endif /* __OCTEONTX_RXTX_H__ */ 313