1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #ifndef _ICE_RXTX_VEC_COMMON_H_ 6 #define _ICE_RXTX_VEC_COMMON_H_ 7 8 #include "../common/rx.h" 9 #include "ice_rxtx.h" 10 11 static inline uint16_t 12 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs, 13 uint16_t nb_bufs, uint8_t *split_flags) 14 { 15 struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/ 16 struct rte_mbuf *start = rxq->pkt_first_seg; 17 struct rte_mbuf *end = rxq->pkt_last_seg; 18 unsigned int pkt_idx, buf_idx; 19 20 for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) { 21 if (end) { 22 /* processing a split packet */ 23 end->next = rx_bufs[buf_idx]; 24 rx_bufs[buf_idx]->data_len += rxq->crc_len; 25 26 start->nb_segs++; 27 start->pkt_len += rx_bufs[buf_idx]->data_len; 28 end = end->next; 29 30 if (!split_flags[buf_idx]) { 31 /* it's the last packet of the set */ 32 start->hash = end->hash; 33 start->vlan_tci = end->vlan_tci; 34 start->ol_flags = end->ol_flags; 35 /* we need to strip crc for the whole packet */ 36 start->pkt_len -= rxq->crc_len; 37 if (end->data_len > rxq->crc_len) { 38 end->data_len -= rxq->crc_len; 39 } else { 40 /* free up last mbuf */ 41 struct rte_mbuf *secondlast = start; 42 43 start->nb_segs--; 44 while (secondlast->next != end) 45 secondlast = secondlast->next; 46 secondlast->data_len -= (rxq->crc_len - 47 end->data_len); 48 secondlast->next = NULL; 49 rte_pktmbuf_free_seg(end); 50 } 51 pkts[pkt_idx++] = start; 52 start = NULL; 53 end = NULL; 54 } 55 } else { 56 /* not processing a split packet */ 57 if (!split_flags[buf_idx]) { 58 /* not a split packet, save and skip */ 59 pkts[pkt_idx++] = rx_bufs[buf_idx]; 60 continue; 61 } 62 start = rx_bufs[buf_idx]; 63 end = start; 64 rx_bufs[buf_idx]->data_len += rxq->crc_len; 65 rx_bufs[buf_idx]->pkt_len += rxq->crc_len; 66 } 67 } 68 69 /* save the partial packet for next time */ 70 rxq->pkt_first_seg = start; 71 rxq->pkt_last_seg = end; 72 memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts))); 73 return pkt_idx; 74 } 75 76 static __rte_always_inline int 77 ice_tx_free_bufs_vec(struct ice_tx_queue *txq) 78 { 79 struct ci_tx_entry *txep; 80 uint32_t n; 81 uint32_t i; 82 int nb_free = 0; 83 struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ]; 84 85 /* check DD bits on threshold descriptor */ 86 if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & 87 rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) != 88 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) 89 return 0; 90 91 n = txq->tx_rs_thresh; 92 93 /* first buffer to free from S/W ring is at index 94 * tx_next_dd - (tx_rs_thresh-1) 95 */ 96 txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)]; 97 m = rte_pktmbuf_prefree_seg(txep[0].mbuf); 98 if (likely(m)) { 99 free[0] = m; 100 nb_free = 1; 101 for (i = 1; i < n; i++) { 102 m = rte_pktmbuf_prefree_seg(txep[i].mbuf); 103 if (likely(m)) { 104 if (likely(m->pool == free[0]->pool)) { 105 free[nb_free++] = m; 106 } else { 107 rte_mempool_put_bulk(free[0]->pool, 108 (void *)free, 109 nb_free); 110 free[0] = m; 111 nb_free = 1; 112 } 113 } 114 } 115 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 116 } else { 117 for (i = 1; i < n; i++) { 118 m = rte_pktmbuf_prefree_seg(txep[i].mbuf); 119 if (m) 120 rte_mempool_put(m->pool, m); 121 } 122 } 123 124 /* buffers were freed, update counters */ 125 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 126 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 127 if (txq->tx_next_dd >= txq->nb_tx_desc) 128 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 129 130 return txq->tx_rs_thresh; 131 } 132 133 static __rte_always_inline void 134 ice_tx_backlog_entry(struct ci_tx_entry *txep, 135 struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 136 { 137 int i; 138 139 for (i = 0; i < (int)nb_pkts; ++i) 140 txep[i].mbuf = tx_pkts[i]; 141 } 142 143 static inline void 144 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq) 145 { 146 const unsigned int mask = rxq->nb_rx_desc - 1; 147 unsigned int i; 148 149 if (unlikely(!rxq->sw_ring)) { 150 PMD_DRV_LOG(DEBUG, "sw_ring is NULL"); 151 return; 152 } 153 154 if (rxq->rxrearm_nb >= rxq->nb_rx_desc) 155 return; 156 157 /* free all mbufs that are valid in the ring */ 158 if (rxq->rxrearm_nb == 0) { 159 for (i = 0; i < rxq->nb_rx_desc; i++) { 160 if (rxq->sw_ring[i].mbuf) 161 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 162 } 163 } else { 164 for (i = rxq->rx_tail; 165 i != rxq->rxrearm_start; 166 i = (i + 1) & mask) { 167 if (rxq->sw_ring[i].mbuf) 168 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 169 } 170 } 171 172 rxq->rxrearm_nb = rxq->nb_rx_desc; 173 174 /* set all entries to NULL */ 175 memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); 176 } 177 178 static inline void 179 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq) 180 { 181 uint16_t i; 182 183 if (unlikely(!txq || !txq->sw_ring)) { 184 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL"); 185 return; 186 } 187 188 /** 189 * vPMD tx will not set sw_ring's mbuf to NULL after free, 190 * so need to free remains more carefully. 191 */ 192 i = txq->tx_next_dd - txq->tx_rs_thresh + 1; 193 194 #ifdef __AVX512VL__ 195 struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id]; 196 197 if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 || 198 dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) { 199 struct ci_tx_entry_vec *swr = (void *)txq->sw_ring; 200 201 if (txq->tx_tail < i) { 202 for (; i < txq->nb_tx_desc; i++) { 203 rte_pktmbuf_free_seg(swr[i].mbuf); 204 swr[i].mbuf = NULL; 205 } 206 i = 0; 207 } 208 for (; i < txq->tx_tail; i++) { 209 rte_pktmbuf_free_seg(swr[i].mbuf); 210 swr[i].mbuf = NULL; 211 } 212 } else 213 #endif 214 { 215 if (txq->tx_tail < i) { 216 for (; i < txq->nb_tx_desc; i++) { 217 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 218 txq->sw_ring[i].mbuf = NULL; 219 } 220 i = 0; 221 } 222 for (; i < txq->tx_tail; i++) { 223 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 224 txq->sw_ring[i].mbuf = NULL; 225 } 226 } 227 } 228 229 static inline int 230 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq) 231 { 232 uintptr_t p; 233 struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ 234 235 mb_def.nb_segs = 1; 236 mb_def.data_off = RTE_PKTMBUF_HEADROOM; 237 mb_def.port = rxq->port_id; 238 rte_mbuf_refcnt_set(&mb_def, 1); 239 240 /* prevent compiler reordering: rearm_data covers previous fields */ 241 rte_compiler_barrier(); 242 p = (uintptr_t)&mb_def.rearm_data; 243 rxq->mbuf_initializer = *(uint64_t *)p; 244 return 0; 245 } 246 247 #define ICE_TX_NO_VECTOR_FLAGS ( \ 248 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \ 249 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | \ 250 RTE_ETH_TX_OFFLOAD_TCP_TSO | \ 251 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | \ 252 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | \ 253 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | \ 254 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | \ 255 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM) 256 257 #define ICE_TX_VECTOR_OFFLOAD ( \ 258 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ 259 RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \ 260 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \ 261 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | \ 262 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ 263 RTE_ETH_TX_OFFLOAD_TCP_CKSUM) 264 265 #define ICE_RX_VECTOR_OFFLOAD ( \ 266 RTE_ETH_RX_OFFLOAD_CHECKSUM | \ 267 RTE_ETH_RX_OFFLOAD_SCTP_CKSUM | \ 268 RTE_ETH_RX_OFFLOAD_VLAN | \ 269 RTE_ETH_RX_OFFLOAD_RSS_HASH) 270 271 #define ICE_VECTOR_PATH 0 272 #define ICE_VECTOR_OFFLOAD_PATH 1 273 274 static inline int 275 ice_rx_vec_queue_default(struct ice_rx_queue *rxq) 276 { 277 if (!rxq) 278 return -1; 279 280 if (!rte_is_power_of_2(rxq->nb_rx_desc)) 281 return -1; 282 283 if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST) 284 return -1; 285 286 if (rxq->nb_rx_desc % rxq->rx_free_thresh) 287 return -1; 288 289 if (rxq->proto_xtr != PROTO_XTR_NONE) 290 return -1; 291 292 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) 293 return -1; 294 295 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) 296 return -1; 297 298 if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD) 299 return ICE_VECTOR_OFFLOAD_PATH; 300 301 return ICE_VECTOR_PATH; 302 } 303 304 static inline int 305 ice_tx_vec_queue_default(struct ice_tx_queue *txq) 306 { 307 if (!txq) 308 return -1; 309 310 if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST || 311 txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ) 312 return -1; 313 314 if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS) 315 return -1; 316 317 if (txq->offloads & ICE_TX_VECTOR_OFFLOAD) 318 return ICE_VECTOR_OFFLOAD_PATH; 319 320 return ICE_VECTOR_PATH; 321 } 322 323 static inline int 324 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev) 325 { 326 int i; 327 struct ice_rx_queue *rxq; 328 int ret = 0; 329 int result = 0; 330 331 for (i = 0; i < dev->data->nb_rx_queues; i++) { 332 rxq = dev->data->rx_queues[i]; 333 ret = (ice_rx_vec_queue_default(rxq)); 334 if (ret < 0) 335 return -1; 336 if (ret == ICE_VECTOR_OFFLOAD_PATH) 337 result = ret; 338 } 339 340 return result; 341 } 342 343 static inline int 344 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev) 345 { 346 int i; 347 struct ice_tx_queue *txq; 348 int ret = 0; 349 int result = 0; 350 351 for (i = 0; i < dev->data->nb_tx_queues; i++) { 352 txq = dev->data->tx_queues[i]; 353 ret = ice_tx_vec_queue_default(txq); 354 if (ret < 0) 355 return -1; 356 if (ret == ICE_VECTOR_OFFLOAD_PATH) 357 result = ret; 358 } 359 360 return result; 361 } 362 363 static inline void 364 ice_txd_enable_offload(struct rte_mbuf *tx_pkt, 365 uint64_t *txd_hi) 366 { 367 uint64_t ol_flags = tx_pkt->ol_flags; 368 uint32_t td_cmd = 0; 369 uint32_t td_offset = 0; 370 371 /* Tx Checksum Offload */ 372 /* SET MACLEN */ 373 td_offset |= (tx_pkt->l2_len >> 1) << 374 ICE_TX_DESC_LEN_MACLEN_S; 375 376 /* Enable L3 checksum offload */ 377 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 378 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 379 td_offset |= (tx_pkt->l3_len >> 2) << 380 ICE_TX_DESC_LEN_IPLEN_S; 381 } else if (ol_flags & RTE_MBUF_F_TX_IPV4) { 382 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 383 td_offset |= (tx_pkt->l3_len >> 2) << 384 ICE_TX_DESC_LEN_IPLEN_S; 385 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) { 386 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 387 td_offset |= (tx_pkt->l3_len >> 2) << 388 ICE_TX_DESC_LEN_IPLEN_S; 389 } 390 391 /* Enable L4 checksum offloads */ 392 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 393 case RTE_MBUF_F_TX_TCP_CKSUM: 394 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 395 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 396 ICE_TX_DESC_LEN_L4_LEN_S; 397 break; 398 case RTE_MBUF_F_TX_SCTP_CKSUM: 399 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 400 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 401 ICE_TX_DESC_LEN_L4_LEN_S; 402 break; 403 case RTE_MBUF_F_TX_UDP_CKSUM: 404 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 405 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << 406 ICE_TX_DESC_LEN_L4_LEN_S; 407 break; 408 default: 409 break; 410 } 411 412 *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S; 413 414 /* Tx VLAN/QINQ insertion Offload */ 415 if (ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { 416 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1; 417 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci << 418 ICE_TXD_QW1_L2TAG1_S); 419 } 420 421 *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S; 422 } 423 #endif 424