1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #ifndef _ICE_RXTX_VEC_COMMON_H_ 6 #define _ICE_RXTX_VEC_COMMON_H_ 7 8 #include "../common/rx.h" 9 #include "ice_rxtx.h" 10 11 static inline uint16_t 12 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs, 13 uint16_t nb_bufs, uint8_t *split_flags) 14 { 15 struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/ 16 struct rte_mbuf *start = rxq->pkt_first_seg; 17 struct rte_mbuf *end = rxq->pkt_last_seg; 18 unsigned int pkt_idx, buf_idx; 19 20 for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { 21 if (end) { 22 /* processing a split packet */ 23 end->next = rx_bufs[buf_idx]; 24 rx_bufs[buf_idx]->data_len += rxq->crc_len; 25 26 start->nb_segs++; 27 start->pkt_len += rx_bufs[buf_idx]->data_len; 28 end = end->next; 29 30 if (!split_flags[buf_idx]) { 31 /* it's the last packet of the set */ 32 start->hash = end->hash; 33 start->vlan_tci = end->vlan_tci; 34 start->ol_flags = end->ol_flags; 35 /* we need to strip crc for the whole packet */ 36 start->pkt_len -= rxq->crc_len; 37 if (end->data_len > rxq->crc_len) { 38 end->data_len -= rxq->crc_len; 39 } else { 40 /* free up last mbuf */ 41 struct rte_mbuf *secondlast = start; 42 43 start->nb_segs--; 44 while (secondlast->next != end) 45 secondlast = secondlast->next; 46 secondlast->data_len -= (rxq->crc_len - 47 end->data_len); 48 secondlast->next = NULL; 49 rte_pktmbuf_free_seg(end); 50 } 51 pkts[pkt_idx++] = start; 52 start = NULL; 53 end = NULL; 54 } 55 } else { 56 /* not processing a split packet */ 57 if (!split_flags[buf_idx]) { 58 /* not a split packet, save and skip */ 59 pkts[pkt_idx++] = rx_bufs[buf_idx]; 60 continue; 61 } 62 start = rx_bufs[buf_idx]; 63 end = start; 64 rx_bufs[buf_idx]->data_len += rxq->crc_len; 65 rx_bufs[buf_idx]->pkt_len += rxq->crc_len; 66 } 67 } 68 69 /* save the partial packet for next time */ 70 rxq->pkt_first_seg = start; 71 rxq->pkt_last_seg = end; 72 memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); 73 return pkt_idx; 74 } 75 76 static inline int 77 ice_tx_desc_done(struct ci_tx_queue *txq, uint16_t idx) 78 { 79 return (txq->ice_tx_ring[idx].cmd_type_offset_bsz & 80 rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) == 81 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE); 82 } 83 84 static __rte_always_inline int 85 ice_tx_free_bufs_vec(struct ci_tx_queue *txq) 86 { 87 return ci_tx_free_bufs(txq, ice_tx_desc_done); 88 } 89 90 static inline void 91 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) 92 { 93 const unsigned int mask = rxq->nb_rx_desc - 1; 94 unsigned int i; 95 96 if (unlikely(!rxq->sw_ring)) { 97 PMD_DRV_LOG(DEBUG, "sw_ring is NULL"); 98 return; 99 } 100 101 if (rxq->rxrearm_nb >= rxq->nb_rx_desc) 102 return; 103 104 /* free all mbufs that are valid in the ring */ 105 if (rxq->rxrearm_nb == 0) { 106 for (i = 0; i < rxq->nb_rx_desc; i++) { 107 if (rxq->sw_ring[i].mbuf) 108 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 109 } 110 } else { 111 for (i = rxq->rx_tail; 112 i != rxq->rxrearm_start; 113 i = (i + 1) & mask) { 114 if (rxq->sw_ring[i].mbuf) 115 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 116 } 117 } 118 119 rxq->rxrearm_nb = rxq->nb_rx_desc; 120 121 /* set all entries to NULL */ 122 memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); 123 } 124 125 static inline void 126 _ice_tx_queue_release_mbufs_vec(struct ci_tx_queue *txq) 127 { 128 uint16_t i; 129 130 if (unlikely(!txq || !txq->sw_ring)) { 131 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); 132 return; 133 } 134 135 /** 136 * vPMD tx will not set sw_ring's mbuf to NULL after free, 137 * so need to free remains more carefully. 138 */ 139 i = txq->tx_next_dd - txq->tx_rs_thresh + 1; 140 141 #ifdef __AVX512VL__ 142 struct rte_eth_dev *dev = &rte_eth_devices[txq->ice_vsi->adapter->pf.dev_data->port_id]; 143 144 if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || 145 dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) { 146 struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; 147 148 if (txq->tx_tail < i) { 149 for (; i < txq->nb_tx_desc; i++) { 150 rte_pktmbuf_free_seg(swr[i].mbuf); 151 swr[i].mbuf = NULL; 152 } 153 i = 0; 154 } 155 for (; i < txq->tx_tail; i++) { 156 rte_pktmbuf_free_seg(swr[i].mbuf); 157 swr[i].mbuf = NULL; 158 } 159 } else 160 #endif 161 { 162 if (txq->tx_tail < i) { 163 for (; i < txq->nb_tx_desc; i++) { 164 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 165 txq->sw_ring[i].mbuf = NULL; 166 } 167 i = 0; 168 } 169 for (; i < txq->tx_tail; i++) { 170 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 171 txq->sw_ring[i].mbuf = NULL; 172 } 173 } 174 } 175 176 static inline int 177 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq) 178 { 179 uintptr_t p; 180 struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ 181 182 mb_def.nb_segs = 1; 183 mb_def.data_off = RTE_PKTMBUF_HEADROOM; 184 mb_def.port = rxq->port_id; 185 rte_mbuf_refcnt_set(&mb_def, 1); 186 187 /* prevent compiler reordering: rearm_data covers previous fields */ 188 rte_compiler_barrier(); 189 p = (uintptr_t)&mb_def.rearm_data; 190 rxq->mbuf_initializer = *(uint64_t *)p; 191 return 0; 192 } 193 194 #define ICE_TX_NO_VECTOR_FLAGS ( \ 195 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \ 196 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | \ 197 RTE_ETH_TX_OFFLOAD_TCP_TSO | \ 198 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | \ 199 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | \ 200 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | \ 201 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | \ 202 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM) 203 204 #define ICE_TX_VECTOR_OFFLOAD ( \ 205 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ 206 RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \ 207 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ 208 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | \ 209 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ 210 RTE_ETH_TX_OFFLOAD_TCP_CKSUM) 211 212 #define ICE_RX_VECTOR_OFFLOAD ( \ 213 RTE_ETH_RX_OFFLOAD_CHECKSUM | \ 214 RTE_ETH_RX_OFFLOAD_SCTP_CKSUM | \ 215 RTE_ETH_RX_OFFLOAD_VLAN | \ 216 RTE_ETH_RX_OFFLOAD_RSS_HASH) 217 218 #define ICE_VECTOR_PATH 0 219 #define ICE_VECTOR_OFFLOAD_PATH 1 220 221 static inline int 222 ice_rx_vec_queue_default(struct ice_rx_queue *rxq) 223 { 224 if (!rxq) 225 return -1; 226 227 if (!rte_is_power_of_2(rxq->nb_rx_desc)) 228 return -1; 229 230 if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST) 231 return -1; 232 233 if (rxq->nb_rx_desc % rxq->rx_free_thresh) 234 return -1; 235 236 if (rxq->proto_xtr != PROTO_XTR_NONE) 237 return -1; 238 239 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) 240 return -1; 241 242 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) 243 return -1; 244 245 if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD) 246 return ICE_VECTOR_OFFLOAD_PATH; 247 248 return ICE_VECTOR_PATH; 249 } 250 251 static inline int 252 ice_tx_vec_queue_default(struct ci_tx_queue *txq) 253 { 254 if (!txq) 255 return -1; 256 257 if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST || 258 txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ) 259 return -1; 260 261 if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS) 262 return -1; 263 264 if (txq->offloads & ICE_TX_VECTOR_OFFLOAD) 265 return ICE_VECTOR_OFFLOAD_PATH; 266 267 return ICE_VECTOR_PATH; 268 } 269 270 static inline int 271 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev) 272 { 273 int i; 274 struct ice_rx_queue *rxq; 275 int ret = 0; 276 int result = 0; 277 278 for (i = 0; i < dev->data->nb_rx_queues; i++) { 279 rxq = dev->data->rx_queues[i]; 280 ret = (ice_rx_vec_queue_default(rxq)); 281 if (ret < 0) 282 return -1; 283 if (ret == ICE_VECTOR_OFFLOAD_PATH) 284 result = ret; 285 } 286 287 return result; 288 } 289 290 static inline int 291 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) 292 { 293 int i; 294 struct ci_tx_queue *txq; 295 int ret = 0; 296 int result = 0; 297 298 for (i = 0; i < dev->data->nb_tx_queues; i++) { 299 txq = dev->data->tx_queues[i]; 300 ret = ice_tx_vec_queue_default(txq); 301 if (ret < 0) 302 return -1; 303 if (ret == ICE_VECTOR_OFFLOAD_PATH) 304 result = ret; 305 } 306 307 return result; 308 } 309 310 static inline void 311 ice_txd_enable_offload(struct rte_mbuf *tx_pkt, 312 uint64_t *txd_hi) 313 { 314 uint64_t ol_flags = tx_pkt->ol_flags; 315 uint32_t td_cmd = 0; 316 uint32_t td_offset = 0; 317 318 /* Tx Checksum Offload */ 319 /* SET MACLEN */ 320 td_offset |= (tx_pkt->l2_len >> 1) << 321 ICE_TX_DESC_LEN_MACLEN_S; 322 323 /* Enable L3 checksum offload */ 324 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 325 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 326 td_offset |= (tx_pkt->l3_len >> 2) << 327 ICE_TX_DESC_LEN_IPLEN_S; 328 } else if (ol_flags & RTE_MBUF_F_TX_IPV4) { 329 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 330 td_offset |= (tx_pkt->l3_len >> 2) << 331 ICE_TX_DESC_LEN_IPLEN_S; 332 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) { 333 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 334 td_offset |= (tx_pkt->l3_len >> 2) << 335 ICE_TX_DESC_LEN_IPLEN_S; 336 } 337 338 /* Enable L4 checksum offloads */ 339 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 340 case RTE_MBUF_F_TX_TCP_CKSUM: 341 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 342 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 343 ICE_TX_DESC_LEN_L4_LEN_S; 344 break; 345 case RTE_MBUF_F_TX_SCTP_CKSUM: 346 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 347 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 348 ICE_TX_DESC_LEN_L4_LEN_S; 349 break; 350 case RTE_MBUF_F_TX_UDP_CKSUM: 351 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 352 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << 353 ICE_TX_DESC_LEN_L4_LEN_S; 354 break; 355 default: 356 break; 357 } 358 359 *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S; 360 361 /* Tx VLAN/QINQ insertion Offload */ 362 if (ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { 363 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1; 364 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << 365 ICE_TXD_QW1_L2TAG1_S); 366 } 367 368 *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S; 369 } 370 #endif 371