1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #ifndef _ICE_RXTX_VEC_COMMON_H_ 6 #define _ICE_RXTX_VEC_COMMON_H_ 7 8 #include "../common/rx.h" 9 #include "ice_rxtx.h" 10 11 static inline uint16_t 12 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs, 13 uint16_t nb_bufs, uint8_t *split_flags) 14 { 15 struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/ 16 struct rte_mbuf *start = rxq->pkt_first_seg; 17 struct rte_mbuf *end = rxq->pkt_last_seg; 18 unsigned int pkt_idx, buf_idx; 19 20 for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { 21 if (end) { 22 /* processing a split packet */ 23 end->next = rx_bufs[buf_idx]; 24 rx_bufs[buf_idx]->data_len += rxq->crc_len; 25 26 start->nb_segs++; 27 start->pkt_len += rx_bufs[buf_idx]->data_len; 28 end = end->next; 29 30 if (!split_flags[buf_idx]) { 31 /* it's the last packet of the set */ 32 start->hash = end->hash; 33 start->vlan_tci = end->vlan_tci; 34 start->ol_flags = end->ol_flags; 35 /* we need to strip crc for the whole packet */ 36 start->pkt_len -= rxq->crc_len; 37 if (end->data_len > rxq->crc_len) { 38 end->data_len -= rxq->crc_len; 39 } else { 40 /* free up last mbuf */ 41 struct rte_mbuf *secondlast = start; 42 43 start->nb_segs--; 44 while (secondlast->next != end) 45 secondlast = secondlast->next; 46 secondlast->data_len -= (rxq->crc_len - 47 end->data_len); 48 secondlast->next = NULL; 49 rte_pktmbuf_free_seg(end); 50 } 51 pkts[pkt_idx++] = start; 52 start = NULL; 53 end = NULL; 54 } 55 } else { 56 /* not processing a split packet */ 57 if (!split_flags[buf_idx]) { 58 /* not a split packet, save and skip */ 59 pkts[pkt_idx++] = rx_bufs[buf_idx]; 60 continue; 61 } 62 start = rx_bufs[buf_idx]; 63 end = start; 64 rx_bufs[buf_idx]->data_len += rxq->crc_len; 65 rx_bufs[buf_idx]->pkt_len += rxq->crc_len; 66 } 67 } 68 69 /* save the partial packet for next time */ 70 rxq->pkt_first_seg = start; 71 rxq->pkt_last_seg = end; 72 memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); 73 return pkt_idx; 74 } 75 76 static __rte_always_inline int 77 ice_tx_free_bufs_vec(struct ice_tx_queue *txq) 78 { 79 struct ci_tx_entry *txep; 80 uint32_t n; 81 uint32_t i; 82 int nb_free = 0; 83 struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ]; 84 85 /* check DD bits on threshold descriptor */ 86 if ((txq->ice_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & 87 rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) != 88 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) 89 return 0; 90 91 n = txq->tx_rs_thresh; 92 93 /* first buffer to free from S/W ring is at index 94 * tx_next_dd - (tx_rs_thresh-1) 95 */ 96 txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; 97 m = rte_pktmbuf_prefree_seg(txep[0].mbuf); 98 if (likely(m)) { 99 free[0] = m; 100 nb_free = 1; 101 for (i = 1; i < n; i++) { 102 m = rte_pktmbuf_prefree_seg(txep[i].mbuf); 103 if (likely(m)) { 104 if (likely(m->pool == free[0]->pool)) { 105 free[nb_free++] = m; 106 } else { 107 rte_mempool_put_bulk(free[0]->pool, 108 (void *)free, 109 nb_free); 110 free[0] = m; 111 nb_free = 1; 112 } 113 } 114 } 115 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 116 } else { 117 for (i = 1; i < n; i++) { 118 m = rte_pktmbuf_prefree_seg(txep[i].mbuf); 119 if (m) 120 rte_mempool_put(m->pool, m); 121 } 122 } 123 124 /* buffers were freed, update counters */ 125 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 126 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 127 if (txq->tx_next_dd >= txq->nb_tx_desc) 128 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 129 130 return txq->tx_rs_thresh; 131 } 132 133 static inline void 134 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) 135 { 136 const unsigned int mask = rxq->nb_rx_desc - 1; 137 unsigned int i; 138 139 if (unlikely(!rxq->sw_ring)) { 140 PMD_DRV_LOG(DEBUG, "sw_ring is NULL"); 141 return; 142 } 143 144 if (rxq->rxrearm_nb >= rxq->nb_rx_desc) 145 return; 146 147 /* free all mbufs that are valid in the ring */ 148 if (rxq->rxrearm_nb == 0) { 149 for (i = 0; i < rxq->nb_rx_desc; i++) { 150 if (rxq->sw_ring[i].mbuf) 151 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 152 } 153 } else { 154 for (i = rxq->rx_tail; 155 i != rxq->rxrearm_start; 156 i = (i + 1) & mask) { 157 if (rxq->sw_ring[i].mbuf) 158 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 159 } 160 } 161 162 rxq->rxrearm_nb = rxq->nb_rx_desc; 163 164 /* set all entries to NULL */ 165 memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); 166 } 167 168 static inline void 169 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) 170 { 171 uint16_t i; 172 173 if (unlikely(!txq || !txq->sw_ring)) { 174 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); 175 return; 176 } 177 178 /** 179 * vPMD tx will not set sw_ring's mbuf to NULL after free, 180 * so need to free remains more carefully. 181 */ 182 i = txq->tx_next_dd - txq->tx_rs_thresh + 1; 183 184 #ifdef __AVX512VL__ 185 struct rte_eth_dev *dev = &rte_eth_devices[txq->ice_vsi->adapter->pf.dev_data->port_id]; 186 187 if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || 188 dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) { 189 struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; 190 191 if (txq->tx_tail < i) { 192 for (; i < txq->nb_tx_desc; i++) { 193 rte_pktmbuf_free_seg(swr[i].mbuf); 194 swr[i].mbuf = NULL; 195 } 196 i = 0; 197 } 198 for (; i < txq->tx_tail; i++) { 199 rte_pktmbuf_free_seg(swr[i].mbuf); 200 swr[i].mbuf = NULL; 201 } 202 } else 203 #endif 204 { 205 if (txq->tx_tail < i) { 206 for (; i < txq->nb_tx_desc; i++) { 207 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 208 txq->sw_ring[i].mbuf = NULL; 209 } 210 i = 0; 211 } 212 for (; i < txq->tx_tail; i++) { 213 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 214 txq->sw_ring[i].mbuf = NULL; 215 } 216 } 217 } 218 219 static inline int 220 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq) 221 { 222 uintptr_t p; 223 struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ 224 225 mb_def.nb_segs = 1; 226 mb_def.data_off = RTE_PKTMBUF_HEADROOM; 227 mb_def.port = rxq->port_id; 228 rte_mbuf_refcnt_set(&mb_def, 1); 229 230 /* prevent compiler reordering: rearm_data covers previous fields */ 231 rte_compiler_barrier(); 232 p = (uintptr_t)&mb_def.rearm_data; 233 rxq->mbuf_initializer = *(uint64_t *)p; 234 return 0; 235 } 236 237 #define ICE_TX_NO_VECTOR_FLAGS ( \ 238 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \ 239 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | \ 240 RTE_ETH_TX_OFFLOAD_TCP_TSO | \ 241 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | \ 242 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | \ 243 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | \ 244 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | \ 245 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM) 246 247 #define ICE_TX_VECTOR_OFFLOAD ( \ 248 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ 249 RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \ 250 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ 251 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | \ 252 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ 253 RTE_ETH_TX_OFFLOAD_TCP_CKSUM) 254 255 #define ICE_RX_VECTOR_OFFLOAD ( \ 256 RTE_ETH_RX_OFFLOAD_CHECKSUM | \ 257 RTE_ETH_RX_OFFLOAD_SCTP_CKSUM | \ 258 RTE_ETH_RX_OFFLOAD_VLAN | \ 259 RTE_ETH_RX_OFFLOAD_RSS_HASH) 260 261 #define ICE_VECTOR_PATH 0 262 #define ICE_VECTOR_OFFLOAD_PATH 1 263 264 static inline int 265 ice_rx_vec_queue_default(struct ice_rx_queue *rxq) 266 { 267 if (!rxq) 268 return -1; 269 270 if (!rte_is_power_of_2(rxq->nb_rx_desc)) 271 return -1; 272 273 if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST) 274 return -1; 275 276 if (rxq->nb_rx_desc % rxq->rx_free_thresh) 277 return -1; 278 279 if (rxq->proto_xtr != PROTO_XTR_NONE) 280 return -1; 281 282 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) 283 return -1; 284 285 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) 286 return -1; 287 288 if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD) 289 return ICE_VECTOR_OFFLOAD_PATH; 290 291 return ICE_VECTOR_PATH; 292 } 293 294 static inline int 295 ice_tx_vec_queue_default(struct ice_tx_queue *txq) 296 { 297 if (!txq) 298 return -1; 299 300 if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST || 301 txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ) 302 return -1; 303 304 if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS) 305 return -1; 306 307 if (txq->offloads & ICE_TX_VECTOR_OFFLOAD) 308 return ICE_VECTOR_OFFLOAD_PATH; 309 310 return ICE_VECTOR_PATH; 311 } 312 313 static inline int 314 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev) 315 { 316 int i; 317 struct ice_rx_queue *rxq; 318 int ret = 0; 319 int result = 0; 320 321 for (i = 0; i < dev->data->nb_rx_queues; i++) { 322 rxq = dev->data->rx_queues[i]; 323 ret = (ice_rx_vec_queue_default(rxq)); 324 if (ret < 0) 325 return -1; 326 if (ret == ICE_VECTOR_OFFLOAD_PATH) 327 result = ret; 328 } 329 330 return result; 331 } 332 333 static inline int 334 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) 335 { 336 int i; 337 struct ice_tx_queue *txq; 338 int ret = 0; 339 int result = 0; 340 341 for (i = 0; i < dev->data->nb_tx_queues; i++) { 342 txq = dev->data->tx_queues[i]; 343 ret = ice_tx_vec_queue_default(txq); 344 if (ret < 0) 345 return -1; 346 if (ret == ICE_VECTOR_OFFLOAD_PATH) 347 result = ret; 348 } 349 350 return result; 351 } 352 353 static inline void 354 ice_txd_enable_offload(struct rte_mbuf *tx_pkt, 355 uint64_t *txd_hi) 356 { 357 uint64_t ol_flags = tx_pkt->ol_flags; 358 uint32_t td_cmd = 0; 359 uint32_t td_offset = 0; 360 361 /* Tx Checksum Offload */ 362 /* SET MACLEN */ 363 td_offset |= (tx_pkt->l2_len >> 1) << 364 ICE_TX_DESC_LEN_MACLEN_S; 365 366 /* Enable L3 checksum offload */ 367 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 368 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 369 td_offset |= (tx_pkt->l3_len >> 2) << 370 ICE_TX_DESC_LEN_IPLEN_S; 371 } else if (ol_flags & RTE_MBUF_F_TX_IPV4) { 372 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 373 td_offset |= (tx_pkt->l3_len >> 2) << 374 ICE_TX_DESC_LEN_IPLEN_S; 375 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) { 376 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 377 td_offset |= (tx_pkt->l3_len >> 2) << 378 ICE_TX_DESC_LEN_IPLEN_S; 379 } 380 381 /* Enable L4 checksum offloads */ 382 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 383 case RTE_MBUF_F_TX_TCP_CKSUM: 384 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 385 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 386 ICE_TX_DESC_LEN_L4_LEN_S; 387 break; 388 case RTE_MBUF_F_TX_SCTP_CKSUM: 389 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 390 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 391 ICE_TX_DESC_LEN_L4_LEN_S; 392 break; 393 case RTE_MBUF_F_TX_UDP_CKSUM: 394 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 395 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << 396 ICE_TX_DESC_LEN_L4_LEN_S; 397 break; 398 default: 399 break; 400 } 401 402 *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S; 403 404 /* Tx VLAN/QINQ insertion Offload */ 405 if (ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { 406 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1; 407 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << 408 ICE_TXD_QW1_L2TAG1_S); 409 } 410 411 *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S; 412 } 413 #endif 414