1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <ethdev_driver.h> 6 #include <rte_net.h> 7 #include <rte_vect.h> 8 9 #include "ice_rxtx.h" 10 #include "ice_rxtx_vec_common.h" 11 12 #define ICE_TX_CKSUM_OFFLOAD_MASK (RTE_MBUF_F_TX_IP_CKSUM | \ 13 RTE_MBUF_F_TX_L4_MASK | \ 14 RTE_MBUF_F_TX_TCP_SEG | \ 15 RTE_MBUF_F_TX_UDP_SEG | \ 16 RTE_MBUF_F_TX_OUTER_IP_CKSUM) 17 18 /** 19 * The mbuf dynamic field pointer for protocol extraction metadata. 20 */ 21 #define ICE_DYNF_PROTO_XTR_METADATA(m, n) \ 22 RTE_MBUF_DYNFIELD((m), (n), uint32_t *) 23 24 static int 25 ice_monitor_callback(const uint64_t value, 26 const uint64_t arg[RTE_POWER_MONITOR_OPAQUE_SZ] __rte_unused) 27 { 28 const uint64_t m = rte_cpu_to_le_16(1 << ICE_RX_FLEX_DESC_STATUS0_DD_S); 29 /* 30 * we expect the DD bit to be set to 1 if this descriptor was already 31 * written to. 32 */ 33 return (value & m) == m ? -1 : 0; 34 } 35 36 int 37 ice_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 38 { 39 volatile union ice_rx_flex_desc *rxdp; 40 struct ice_rx_queue *rxq = rx_queue; 41 uint16_t desc; 42 43 desc = rxq->rx_tail; 44 rxdp = &rxq->rx_ring[desc]; 45 /* watch for changes in status bit */ 46 pmc->addr = &rxdp->wb.status_error0; 47 48 /* comparison callback */ 49 pmc->fn = ice_monitor_callback; 50 51 /* register is 16-bit */ 52 pmc->size = sizeof(uint16_t); 53 54 return 0; 55 } 56 57 58 static inline uint8_t 59 ice_proto_xtr_type_to_rxdid(uint8_t xtr_type) 60 { 61 static uint8_t rxdid_map[] = { 62 [PROTO_XTR_NONE] = ICE_RXDID_COMMS_OVS, 63 [PROTO_XTR_VLAN] = ICE_RXDID_COMMS_AUX_VLAN, 64 [PROTO_XTR_IPV4] = ICE_RXDID_COMMS_AUX_IPV4, 65 [PROTO_XTR_IPV6] = ICE_RXDID_COMMS_AUX_IPV6, 66 [PROTO_XTR_IPV6_FLOW] = ICE_RXDID_COMMS_AUX_IPV6_FLOW, 67 [PROTO_XTR_TCP] = ICE_RXDID_COMMS_AUX_TCP, 68 [PROTO_XTR_IP_OFFSET] = ICE_RXDID_COMMS_AUX_IP_OFFSET, 69 }; 70 71 return xtr_type < RTE_DIM(rxdid_map) ? 72 rxdid_map[xtr_type] : ICE_RXDID_COMMS_OVS; 73 } 74 75 static inline void 76 ice_rxd_to_pkt_fields_by_comms_generic(__rte_unused struct ice_rx_queue *rxq, 77 struct rte_mbuf *mb, 78 volatile union ice_rx_flex_desc *rxdp) 79 { 80 volatile struct ice_32b_rx_flex_desc_comms *desc = 81 (volatile struct ice_32b_rx_flex_desc_comms *)rxdp; 82 uint16_t stat_err = rte_le_to_cpu_16(desc->status_error0); 83 84 if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 85 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 86 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 87 } 88 89 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 90 if (desc->flow_id != 0xFFFFFFFF) { 91 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 92 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 93 } 94 #endif 95 } 96 97 static inline void 98 ice_rxd_to_pkt_fields_by_comms_ovs(__rte_unused struct ice_rx_queue *rxq, 99 struct rte_mbuf *mb, 100 volatile union ice_rx_flex_desc *rxdp) 101 { 102 volatile struct ice_32b_rx_flex_desc_comms_ovs *desc = 103 (volatile struct ice_32b_rx_flex_desc_comms_ovs *)rxdp; 104 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 105 uint16_t stat_err; 106 #endif 107 108 if (desc->flow_id != 0xFFFFFFFF) { 109 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 110 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 111 } 112 113 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 114 stat_err = rte_le_to_cpu_16(desc->status_error0); 115 if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 116 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 117 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 118 } 119 #endif 120 } 121 122 static inline void 123 ice_rxd_to_pkt_fields_by_comms_aux_v1(struct ice_rx_queue *rxq, 124 struct rte_mbuf *mb, 125 volatile union ice_rx_flex_desc *rxdp) 126 { 127 volatile struct ice_32b_rx_flex_desc_comms *desc = 128 (volatile struct ice_32b_rx_flex_desc_comms *)rxdp; 129 uint16_t stat_err; 130 131 stat_err = rte_le_to_cpu_16(desc->status_error0); 132 if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 133 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 134 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 135 } 136 137 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 138 if (desc->flow_id != 0xFFFFFFFF) { 139 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 140 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 141 } 142 143 if (rxq->xtr_ol_flag) { 144 uint32_t metadata = 0; 145 146 stat_err = rte_le_to_cpu_16(desc->status_error1); 147 148 if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S)) 149 metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0); 150 151 if (stat_err & (1 << ICE_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S)) 152 metadata |= 153 rte_le_to_cpu_16(desc->flex_ts.flex.aux1) << 16; 154 155 if (metadata) { 156 mb->ol_flags |= rxq->xtr_ol_flag; 157 158 *ICE_DYNF_PROTO_XTR_METADATA(mb, rxq->xtr_field_offs) = metadata; 159 } 160 } 161 #else 162 RTE_SET_USED(rxq); 163 #endif 164 } 165 166 static inline void 167 ice_rxd_to_pkt_fields_by_comms_aux_v2(struct ice_rx_queue *rxq, 168 struct rte_mbuf *mb, 169 volatile union ice_rx_flex_desc *rxdp) 170 { 171 volatile struct ice_32b_rx_flex_desc_comms *desc = 172 (volatile struct ice_32b_rx_flex_desc_comms *)rxdp; 173 uint16_t stat_err; 174 175 stat_err = rte_le_to_cpu_16(desc->status_error0); 176 if (likely(stat_err & (1 << ICE_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { 177 mb->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 178 mb->hash.rss = rte_le_to_cpu_32(desc->rss_hash); 179 } 180 181 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 182 if (desc->flow_id != 0xFFFFFFFF) { 183 mb->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID; 184 mb->hash.fdir.hi = rte_le_to_cpu_32(desc->flow_id); 185 } 186 187 if (rxq->xtr_ol_flag) { 188 uint32_t metadata = 0; 189 190 if (desc->flex_ts.flex.aux0 != 0xFFFF) 191 metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux0); 192 else if (desc->flex_ts.flex.aux1 != 0xFFFF) 193 metadata = rte_le_to_cpu_16(desc->flex_ts.flex.aux1); 194 195 if (metadata) { 196 mb->ol_flags |= rxq->xtr_ol_flag; 197 198 *ICE_DYNF_PROTO_XTR_METADATA(mb, rxq->xtr_field_offs) = metadata; 199 } 200 } 201 #else 202 RTE_SET_USED(rxq); 203 #endif 204 } 205 206 static const ice_rxd_to_pkt_fields_t rxd_to_pkt_fields_ops[] = { 207 [ICE_RXDID_COMMS_AUX_VLAN] = ice_rxd_to_pkt_fields_by_comms_aux_v1, 208 [ICE_RXDID_COMMS_AUX_IPV4] = ice_rxd_to_pkt_fields_by_comms_aux_v1, 209 [ICE_RXDID_COMMS_AUX_IPV6] = ice_rxd_to_pkt_fields_by_comms_aux_v1, 210 [ICE_RXDID_COMMS_AUX_IPV6_FLOW] = ice_rxd_to_pkt_fields_by_comms_aux_v1, 211 [ICE_RXDID_COMMS_AUX_TCP] = ice_rxd_to_pkt_fields_by_comms_aux_v1, 212 [ICE_RXDID_COMMS_AUX_IP_OFFSET] = ice_rxd_to_pkt_fields_by_comms_aux_v2, 213 [ICE_RXDID_COMMS_GENERIC] = ice_rxd_to_pkt_fields_by_comms_generic, 214 [ICE_RXDID_COMMS_OVS] = ice_rxd_to_pkt_fields_by_comms_ovs, 215 }; 216 217 void 218 ice_select_rxd_to_pkt_fields_handler(struct ice_rx_queue *rxq, uint32_t rxdid) 219 { 220 rxq->rxdid = rxdid; 221 222 switch (rxdid) { 223 case ICE_RXDID_COMMS_AUX_VLAN: 224 case ICE_RXDID_COMMS_AUX_IPV4: 225 case ICE_RXDID_COMMS_AUX_IPV6: 226 case ICE_RXDID_COMMS_AUX_IPV6_FLOW: 227 case ICE_RXDID_COMMS_AUX_TCP: 228 case ICE_RXDID_COMMS_AUX_IP_OFFSET: 229 break; 230 case ICE_RXDID_COMMS_GENERIC: 231 /* fallthrough */ 232 case ICE_RXDID_COMMS_OVS: 233 break; 234 235 default: 236 /* update this according to the RXDID for PROTO_XTR_NONE */ 237 rxq->rxdid = ICE_RXDID_COMMS_OVS; 238 break; 239 } 240 241 if (rxq->xtr_field_offs == -1) 242 rxq->xtr_ol_flag = 0; 243 } 244 245 static int 246 ice_program_hw_rx_queue(struct ice_rx_queue *rxq) 247 { 248 struct ice_vsi *vsi = rxq->vsi; 249 struct ice_hw *hw = ICE_VSI_TO_HW(vsi); 250 struct ice_pf *pf = ICE_VSI_TO_PF(vsi); 251 struct rte_eth_dev_data *dev_data = rxq->vsi->adapter->pf.dev_data; 252 struct ice_rlan_ctx rx_ctx; 253 uint16_t buf_size; 254 uint32_t rxdid = ICE_RXDID_COMMS_OVS; 255 uint32_t regval; 256 struct ice_adapter *ad = rxq->vsi->adapter; 257 uint32_t frame_size = dev_data->mtu + ICE_ETH_OVERHEAD; 258 int err; 259 260 /* Set buffer size as the head split is disabled. */ 261 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 262 RTE_PKTMBUF_HEADROOM); 263 rxq->rx_buf_len = RTE_ALIGN_FLOOR(buf_size, (1 << ICE_RLAN_CTX_DBUF_S)); 264 rxq->rx_buf_len = RTE_MIN(rxq->rx_buf_len, ICE_RX_MAX_DATA_BUF_SIZE); 265 rxq->max_pkt_len = 266 RTE_MIN((uint32_t)ICE_SUPPORT_CHAIN_NUM * rxq->rx_buf_len, 267 frame_size); 268 269 if (rxq->max_pkt_len <= RTE_ETHER_MIN_LEN || 270 rxq->max_pkt_len > ICE_FRAME_SIZE_MAX) { 271 PMD_DRV_LOG(ERR, "maximum packet length must " 272 "be larger than %u and smaller than %u", 273 (uint32_t)RTE_ETHER_MIN_LEN, 274 (uint32_t)ICE_FRAME_SIZE_MAX); 275 return -EINVAL; 276 } 277 278 if (!rxq->ts_enable && (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { 279 /* Register mbuf field and flag for Rx timestamp */ 280 err = rte_mbuf_dyn_rx_timestamp_register( 281 &ice_timestamp_dynfield_offset, 282 &ice_timestamp_dynflag); 283 if (err) { 284 PMD_DRV_LOG(ERR, 285 "Cannot register mbuf field/flag for timestamp"); 286 return -EINVAL; 287 } 288 rxq->ts_enable = true; 289 } 290 291 memset(&rx_ctx, 0, sizeof(rx_ctx)); 292 293 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) { 294 uint32_t proto_hdr; 295 proto_hdr = rxq->rxseg[0].proto_hdr; 296 297 if (proto_hdr == RTE_PTYPE_UNKNOWN) { 298 PMD_DRV_LOG(ERR, "Buffer split protocol must be configured"); 299 return -EINVAL; 300 } 301 302 switch (proto_hdr & RTE_PTYPE_L4_MASK) { 303 case RTE_PTYPE_L4_TCP: 304 case RTE_PTYPE_L4_UDP: 305 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 306 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP; 307 goto set_hsplit_finish; 308 case RTE_PTYPE_L4_SCTP: 309 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 310 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP; 311 goto set_hsplit_finish; 312 } 313 314 switch (proto_hdr & RTE_PTYPE_L3_MASK) { 315 case RTE_PTYPE_L3_IPV4_EXT_UNKNOWN: 316 case RTE_PTYPE_L3_IPV6_EXT_UNKNOWN: 317 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 318 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_IP; 319 goto set_hsplit_finish; 320 } 321 322 switch (proto_hdr & RTE_PTYPE_L2_MASK) { 323 case RTE_PTYPE_L2_ETHER: 324 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 325 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2; 326 rx_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_SPLIT_L2; 327 goto set_hsplit_finish; 328 } 329 330 switch (proto_hdr & RTE_PTYPE_INNER_L4_MASK) { 331 case RTE_PTYPE_INNER_L4_TCP: 332 case RTE_PTYPE_INNER_L4_UDP: 333 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 334 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP; 335 goto set_hsplit_finish; 336 case RTE_PTYPE_INNER_L4_SCTP: 337 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 338 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP; 339 goto set_hsplit_finish; 340 } 341 342 switch (proto_hdr & RTE_PTYPE_INNER_L3_MASK) { 343 case RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN: 344 case RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN: 345 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 346 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_IP; 347 goto set_hsplit_finish; 348 } 349 350 switch (proto_hdr & RTE_PTYPE_INNER_L2_MASK) { 351 case RTE_PTYPE_INNER_L2_ETHER: 352 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 353 rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2; 354 goto set_hsplit_finish; 355 } 356 357 switch (proto_hdr & RTE_PTYPE_TUNNEL_MASK) { 358 case RTE_PTYPE_TUNNEL_GRENAT: 359 rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT; 360 rx_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_SPLIT_ALWAYS; 361 goto set_hsplit_finish; 362 } 363 364 PMD_DRV_LOG(ERR, "Buffer split protocol is not supported"); 365 return -EINVAL; 366 367 set_hsplit_finish: 368 rxq->rx_hdr_len = ICE_RX_HDR_BUF_SIZE; 369 } else { 370 rxq->rx_hdr_len = 0; 371 rx_ctx.dtype = 0; /* No Protocol Based Buffer Split mode */ 372 } 373 374 rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT; 375 rx_ctx.qlen = rxq->nb_rx_desc; 376 rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S; 377 rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S; 378 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 379 rx_ctx.dsize = 1; /* 32B descriptors */ 380 #endif 381 rx_ctx.rxmax = rxq->max_pkt_len; 382 /* TPH: Transaction Layer Packet (TLP) processing hints */ 383 rx_ctx.tphrdesc_ena = 1; 384 rx_ctx.tphwdesc_ena = 1; 385 rx_ctx.tphdata_ena = 1; 386 rx_ctx.tphhead_ena = 1; 387 /* Low Receive Queue Threshold defined in 64 descriptors units. 388 * When the number of free descriptors goes below the lrxqthresh, 389 * an immediate interrupt is triggered. 390 */ 391 rx_ctx.lrxqthresh = 2; 392 /*default use 32 byte descriptor, vlan tag extract to L2TAG2(1st)*/ 393 rx_ctx.l2tsel = 1; 394 rx_ctx.showiv = 0; 395 rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0; 396 397 rxdid = ice_proto_xtr_type_to_rxdid(rxq->proto_xtr); 398 399 PMD_DRV_LOG(DEBUG, "Port (%u) - Rx queue (%u) is set with RXDID : %u", 400 rxq->port_id, rxq->queue_id, rxdid); 401 402 if (!(pf->supported_rxdid & BIT(rxdid))) { 403 PMD_DRV_LOG(ERR, "currently package doesn't support RXDID (%u)", 404 rxdid); 405 return -EINVAL; 406 } 407 408 rxq->rxdid = rxdid; 409 410 /* Enable Flexible Descriptors in the queue context which 411 * allows this driver to select a specific receive descriptor format 412 */ 413 regval = (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) & 414 QRXFLXP_CNTXT_RXDID_IDX_M; 415 416 /* increasing context priority to pick up profile ID; 417 * default is 0x01; setting to 0x03 to ensure profile 418 * is programming if prev context is of same priority 419 */ 420 regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) & 421 QRXFLXP_CNTXT_RXDID_PRIO_M; 422 423 if (ad->ptp_ena || rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) 424 regval |= QRXFLXP_CNTXT_TS_M; 425 426 ICE_WRITE_REG(hw, QRXFLXP_CNTXT(rxq->reg_idx), regval); 427 428 err = ice_clear_rxq_ctx(hw, rxq->reg_idx); 429 if (err) { 430 PMD_DRV_LOG(ERR, "Failed to clear Lan Rx queue (%u) context", 431 rxq->queue_id); 432 return -EINVAL; 433 } 434 err = ice_write_rxq_ctx(hw, &rx_ctx, rxq->reg_idx); 435 if (err) { 436 PMD_DRV_LOG(ERR, "Failed to write Lan Rx queue (%u) context", 437 rxq->queue_id); 438 return -EINVAL; 439 } 440 441 /* Check if scattered RX needs to be used. */ 442 if (frame_size > buf_size) 443 dev_data->scattered_rx = 1; 444 445 rxq->qrx_tail = hw->hw_addr + QRX_TAIL(rxq->reg_idx); 446 447 /* Init the Rx tail register*/ 448 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 449 450 return 0; 451 } 452 453 /* Allocate mbufs for all descriptors in rx queue */ 454 static int 455 ice_alloc_rx_queue_mbufs(struct ice_rx_queue *rxq) 456 { 457 struct ice_rx_entry *rxe = rxq->sw_ring; 458 uint64_t dma_addr; 459 uint16_t i; 460 461 for (i = 0; i < rxq->nb_rx_desc; i++) { 462 volatile union ice_rx_flex_desc *rxd; 463 rxd = &rxq->rx_ring[i]; 464 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp); 465 466 if (unlikely(!mbuf)) { 467 PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX"); 468 return -ENOMEM; 469 } 470 471 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 472 mbuf->nb_segs = 1; 473 mbuf->port = rxq->port_id; 474 475 dma_addr = 476 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 477 478 if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 479 rte_mbuf_refcnt_set(mbuf, 1); 480 mbuf->next = NULL; 481 rxd->read.hdr_addr = 0; 482 rxd->read.pkt_addr = dma_addr; 483 } else { 484 struct rte_mbuf *mbuf_pay; 485 mbuf_pay = rte_mbuf_raw_alloc(rxq->rxseg[1].mp); 486 if (unlikely(!mbuf_pay)) { 487 rte_pktmbuf_free(mbuf); 488 PMD_DRV_LOG(ERR, "Failed to allocate payload mbuf for RX"); 489 return -ENOMEM; 490 } 491 492 mbuf_pay->next = NULL; 493 mbuf_pay->data_off = RTE_PKTMBUF_HEADROOM; 494 mbuf_pay->nb_segs = 1; 495 mbuf_pay->port = rxq->port_id; 496 mbuf->next = mbuf_pay; 497 498 rxd->read.hdr_addr = dma_addr; 499 /* The LS bit should be set to zero regardless of 500 * buffer split enablement. 501 */ 502 rxd->read.pkt_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf_pay)); 503 } 504 505 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 506 rxd->read.rsvd1 = 0; 507 rxd->read.rsvd2 = 0; 508 #endif 509 rxe[i].mbuf = mbuf; 510 } 511 512 return 0; 513 } 514 515 /* Free all mbufs for descriptors in rx queue */ 516 static void 517 _ice_rx_queue_release_mbufs(struct ice_rx_queue *rxq) 518 { 519 uint16_t i; 520 521 if (!rxq || !rxq->sw_ring) { 522 PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL"); 523 return; 524 } 525 526 for (i = 0; i < rxq->nb_rx_desc; i++) { 527 if (rxq->sw_ring[i].mbuf) { 528 rte_pktmbuf_free(rxq->sw_ring[i].mbuf); 529 rxq->sw_ring[i].mbuf = NULL; 530 } 531 } 532 if (rxq->rx_nb_avail == 0) 533 return; 534 for (i = 0; i < rxq->rx_nb_avail; i++) 535 rte_pktmbuf_free(rxq->rx_stage[rxq->rx_next_avail + i]); 536 537 rxq->rx_nb_avail = 0; 538 } 539 540 /* turn on or off rx queue 541 * @q_idx: queue index in pf scope 542 * @on: turn on or off the queue 543 */ 544 static int 545 ice_switch_rx_queue(struct ice_hw *hw, uint16_t q_idx, bool on) 546 { 547 uint32_t reg; 548 uint16_t j; 549 550 /* QRX_CTRL = QRX_ENA */ 551 reg = ICE_READ_REG(hw, QRX_CTRL(q_idx)); 552 553 if (on) { 554 if (reg & QRX_CTRL_QENA_STAT_M) 555 return 0; /* Already on, skip */ 556 reg |= QRX_CTRL_QENA_REQ_M; 557 } else { 558 if (!(reg & QRX_CTRL_QENA_STAT_M)) 559 return 0; /* Already off, skip */ 560 reg &= ~QRX_CTRL_QENA_REQ_M; 561 } 562 563 /* Write the register */ 564 ICE_WRITE_REG(hw, QRX_CTRL(q_idx), reg); 565 /* Check the result. It is said that QENA_STAT 566 * follows the QENA_REQ not more than 10 use. 567 * TODO: need to change the wait counter later 568 */ 569 for (j = 0; j < ICE_CHK_Q_ENA_COUNT; j++) { 570 rte_delay_us(ICE_CHK_Q_ENA_INTERVAL_US); 571 reg = ICE_READ_REG(hw, QRX_CTRL(q_idx)); 572 if (on) { 573 if ((reg & QRX_CTRL_QENA_REQ_M) && 574 (reg & QRX_CTRL_QENA_STAT_M)) 575 break; 576 } else { 577 if (!(reg & QRX_CTRL_QENA_REQ_M) && 578 !(reg & QRX_CTRL_QENA_STAT_M)) 579 break; 580 } 581 } 582 583 /* Check if it is timeout */ 584 if (j >= ICE_CHK_Q_ENA_COUNT) { 585 PMD_DRV_LOG(ERR, "Failed to %s rx queue[%u]", 586 (on ? "enable" : "disable"), q_idx); 587 return -ETIMEDOUT; 588 } 589 590 return 0; 591 } 592 593 static inline int 594 ice_check_rx_burst_bulk_alloc_preconditions(struct ice_rx_queue *rxq) 595 { 596 int ret = 0; 597 598 if (!(rxq->rx_free_thresh >= ICE_RX_MAX_BURST)) { 599 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 600 "rxq->rx_free_thresh=%d, " 601 "ICE_RX_MAX_BURST=%d", 602 rxq->rx_free_thresh, ICE_RX_MAX_BURST); 603 ret = -EINVAL; 604 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 605 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 606 "rxq->rx_free_thresh=%d, " 607 "rxq->nb_rx_desc=%d", 608 rxq->rx_free_thresh, rxq->nb_rx_desc); 609 ret = -EINVAL; 610 } else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) { 611 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 612 "rxq->nb_rx_desc=%d, " 613 "rxq->rx_free_thresh=%d", 614 rxq->nb_rx_desc, rxq->rx_free_thresh); 615 ret = -EINVAL; 616 } 617 618 return ret; 619 } 620 621 /* reset fields in ice_rx_queue back to default */ 622 static void 623 ice_reset_rx_queue(struct ice_rx_queue *rxq) 624 { 625 unsigned int i; 626 uint16_t len; 627 628 if (!rxq) { 629 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL"); 630 return; 631 } 632 633 len = (uint16_t)(rxq->nb_rx_desc + ICE_RX_MAX_BURST); 634 635 for (i = 0; i < len * sizeof(union ice_rx_flex_desc); i++) 636 ((volatile char *)rxq->rx_ring)[i] = 0; 637 638 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 639 for (i = 0; i < ICE_RX_MAX_BURST; ++i) 640 rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf; 641 642 rxq->rx_nb_avail = 0; 643 rxq->rx_next_avail = 0; 644 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 645 646 rxq->rx_tail = 0; 647 rxq->nb_rx_hold = 0; 648 rxq->pkt_first_seg = NULL; 649 rxq->pkt_last_seg = NULL; 650 651 rxq->rxrearm_start = 0; 652 rxq->rxrearm_nb = 0; 653 } 654 655 int 656 ice_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 657 { 658 struct ice_rx_queue *rxq; 659 int err; 660 struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 661 662 PMD_INIT_FUNC_TRACE(); 663 664 if (rx_queue_id >= dev->data->nb_rx_queues) { 665 PMD_DRV_LOG(ERR, "RX queue %u is out of range %u", 666 rx_queue_id, dev->data->nb_rx_queues); 667 return -EINVAL; 668 } 669 670 rxq = dev->data->rx_queues[rx_queue_id]; 671 if (!rxq || !rxq->q_set) { 672 PMD_DRV_LOG(ERR, "RX queue %u not available or setup", 673 rx_queue_id); 674 return -EINVAL; 675 } 676 677 if (dev->data->rx_queue_state[rx_queue_id] == 678 RTE_ETH_QUEUE_STATE_STARTED) 679 return 0; 680 681 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) 682 rxq->ts_enable = true; 683 err = ice_program_hw_rx_queue(rxq); 684 if (err) { 685 PMD_DRV_LOG(ERR, "fail to program RX queue %u", 686 rx_queue_id); 687 return -EIO; 688 } 689 690 err = ice_alloc_rx_queue_mbufs(rxq); 691 if (err) { 692 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); 693 return -ENOMEM; 694 } 695 696 /* Init the RX tail register. */ 697 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 698 699 err = ice_switch_rx_queue(hw, rxq->reg_idx, true); 700 if (err) { 701 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", 702 rx_queue_id); 703 704 rxq->rx_rel_mbufs(rxq); 705 ice_reset_rx_queue(rxq); 706 return -EINVAL; 707 } 708 709 dev->data->rx_queue_state[rx_queue_id] = 710 RTE_ETH_QUEUE_STATE_STARTED; 711 712 return 0; 713 } 714 715 int 716 ice_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 717 { 718 struct ice_rx_queue *rxq; 719 int err; 720 struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 721 722 if (rx_queue_id < dev->data->nb_rx_queues) { 723 rxq = dev->data->rx_queues[rx_queue_id]; 724 725 if (dev->data->rx_queue_state[rx_queue_id] == 726 RTE_ETH_QUEUE_STATE_STOPPED) 727 return 0; 728 729 err = ice_switch_rx_queue(hw, rxq->reg_idx, false); 730 if (err) { 731 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", 732 rx_queue_id); 733 return -EINVAL; 734 } 735 rxq->rx_rel_mbufs(rxq); 736 ice_reset_rx_queue(rxq); 737 dev->data->rx_queue_state[rx_queue_id] = 738 RTE_ETH_QUEUE_STATE_STOPPED; 739 } 740 741 return 0; 742 } 743 744 int 745 ice_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 746 { 747 struct ci_tx_queue *txq; 748 int err; 749 struct ice_vsi *vsi; 750 struct ice_hw *hw; 751 struct ice_pf *pf; 752 struct ice_aqc_add_tx_qgrp *txq_elem; 753 struct ice_tlan_ctx tx_ctx; 754 int buf_len; 755 struct ice_adapter *ad = ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 756 757 PMD_INIT_FUNC_TRACE(); 758 759 if (tx_queue_id >= dev->data->nb_tx_queues) { 760 PMD_DRV_LOG(ERR, "TX queue %u is out of range %u", 761 tx_queue_id, dev->data->nb_tx_queues); 762 return -EINVAL; 763 } 764 765 txq = dev->data->tx_queues[tx_queue_id]; 766 if (!txq || !txq->q_set) { 767 PMD_DRV_LOG(ERR, "TX queue %u is not available or setup", 768 tx_queue_id); 769 return -EINVAL; 770 } 771 772 if (dev->data->tx_queue_state[tx_queue_id] == 773 RTE_ETH_QUEUE_STATE_STARTED) 774 return 0; 775 776 buf_len = ice_struct_size(txq_elem, txqs, 1); 777 txq_elem = ice_malloc(hw, buf_len); 778 if (!txq_elem) 779 return -ENOMEM; 780 781 vsi = txq->ice_vsi; 782 hw = ICE_VSI_TO_HW(vsi); 783 pf = ICE_VSI_TO_PF(vsi); 784 785 memset(&tx_ctx, 0, sizeof(tx_ctx)); 786 txq_elem->num_txqs = 1; 787 txq_elem->txqs[0].txq_id = rte_cpu_to_le_16(txq->reg_idx); 788 789 tx_ctx.base = txq->tx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT; 790 tx_ctx.qlen = txq->nb_tx_desc; 791 tx_ctx.pf_num = hw->pf_id; 792 tx_ctx.vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; 793 tx_ctx.src_vsi = vsi->vsi_id; 794 tx_ctx.port_num = hw->port_info->lport; 795 tx_ctx.tso_ena = 1; /* tso enable */ 796 tx_ctx.tso_qnum = txq->reg_idx; /* index for tso state structure */ 797 tx_ctx.legacy_int = 1; /* Legacy or Advanced Host Interface */ 798 tx_ctx.tsyn_ena = 1; 799 800 ice_set_ctx(hw, (uint8_t *)&tx_ctx, txq_elem->txqs[0].txq_ctx, 801 ice_tlan_ctx_info); 802 803 txq->qtx_tail = hw->hw_addr + QTX_COMM_DBELL(txq->reg_idx); 804 805 /* Init the Tx tail register*/ 806 ICE_PCI_REG_WRITE(txq->qtx_tail, 0); 807 808 /* Fix me, we assume TC always 0 here */ 809 err = ice_ena_vsi_txq(hw->port_info, vsi->idx, 0, tx_queue_id, 1, 810 txq_elem, buf_len, NULL); 811 if (err) { 812 PMD_DRV_LOG(ERR, "Failed to add lan txq"); 813 rte_free(txq_elem); 814 return -EIO; 815 } 816 /* store the schedule node id */ 817 txq->q_teid = txq_elem->txqs[0].q_teid; 818 819 /* move the queue to correct position in hierarchy, if explicit hierarchy configured */ 820 if (pf->tm_conf.committed) 821 if (ice_tm_setup_txq_node(pf, hw, tx_queue_id, txq->q_teid) != 0) { 822 PMD_DRV_LOG(ERR, "Failed to set up txq traffic management node"); 823 rte_free(txq_elem); 824 return -EIO; 825 } 826 827 /* record what kind of descriptor cleanup we need on teardown */ 828 txq->vector_tx = ad->tx_vec_allowed; 829 830 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 831 832 rte_free(txq_elem); 833 return 0; 834 } 835 836 static int 837 ice_fdir_program_hw_rx_queue(struct ice_rx_queue *rxq) 838 { 839 struct ice_vsi *vsi = rxq->vsi; 840 struct ice_hw *hw = ICE_VSI_TO_HW(vsi); 841 uint32_t rxdid = ICE_RXDID_LEGACY_1; 842 struct ice_rlan_ctx rx_ctx; 843 uint32_t regval; 844 int err; 845 846 rxq->rx_hdr_len = 0; 847 rxq->rx_buf_len = 1024; 848 849 memset(&rx_ctx, 0, sizeof(rx_ctx)); 850 851 rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT; 852 rx_ctx.qlen = rxq->nb_rx_desc; 853 rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S; 854 rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S; 855 rx_ctx.dtype = 0; /* No Buffer Split mode */ 856 rx_ctx.dsize = 1; /* 32B descriptors */ 857 rx_ctx.rxmax = ICE_ETH_MAX_LEN; 858 /* TPH: Transaction Layer Packet (TLP) processing hints */ 859 rx_ctx.tphrdesc_ena = 1; 860 rx_ctx.tphwdesc_ena = 1; 861 rx_ctx.tphdata_ena = 1; 862 rx_ctx.tphhead_ena = 1; 863 /* Low Receive Queue Threshold defined in 64 descriptors units. 864 * When the number of free descriptors goes below the lrxqthresh, 865 * an immediate interrupt is triggered. 866 */ 867 rx_ctx.lrxqthresh = 2; 868 /*default use 32 byte descriptor, vlan tag extract to L2TAG2(1st)*/ 869 rx_ctx.l2tsel = 1; 870 rx_ctx.showiv = 0; 871 rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0; 872 873 /* Enable Flexible Descriptors in the queue context which 874 * allows this driver to select a specific receive descriptor format 875 */ 876 regval = (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) & 877 QRXFLXP_CNTXT_RXDID_IDX_M; 878 879 /* increasing context priority to pick up profile ID; 880 * default is 0x01; setting to 0x03 to ensure profile 881 * is programming if prev context is of same priority 882 */ 883 regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) & 884 QRXFLXP_CNTXT_RXDID_PRIO_M; 885 886 ICE_WRITE_REG(hw, QRXFLXP_CNTXT(rxq->reg_idx), regval); 887 888 err = ice_clear_rxq_ctx(hw, rxq->reg_idx); 889 if (err) { 890 PMD_DRV_LOG(ERR, "Failed to clear Lan Rx queue (%u) context", 891 rxq->queue_id); 892 return -EINVAL; 893 } 894 err = ice_write_rxq_ctx(hw, &rx_ctx, rxq->reg_idx); 895 if (err) { 896 PMD_DRV_LOG(ERR, "Failed to write Lan Rx queue (%u) context", 897 rxq->queue_id); 898 return -EINVAL; 899 } 900 901 rxq->qrx_tail = hw->hw_addr + QRX_TAIL(rxq->reg_idx); 902 903 /* Init the Rx tail register*/ 904 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 905 906 return 0; 907 } 908 909 int 910 ice_fdir_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 911 { 912 struct ice_rx_queue *rxq; 913 int err; 914 struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 915 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 916 917 PMD_INIT_FUNC_TRACE(); 918 919 rxq = pf->fdir.rxq; 920 if (!rxq || !rxq->q_set) { 921 PMD_DRV_LOG(ERR, "FDIR RX queue %u not available or setup", 922 rx_queue_id); 923 return -EINVAL; 924 } 925 926 err = ice_fdir_program_hw_rx_queue(rxq); 927 if (err) { 928 PMD_DRV_LOG(ERR, "fail to program FDIR RX queue %u", 929 rx_queue_id); 930 return -EIO; 931 } 932 933 /* Init the RX tail register. */ 934 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 935 936 err = ice_switch_rx_queue(hw, rxq->reg_idx, true); 937 if (err) { 938 PMD_DRV_LOG(ERR, "Failed to switch FDIR RX queue %u on", 939 rx_queue_id); 940 941 ice_reset_rx_queue(rxq); 942 return -EINVAL; 943 } 944 945 return 0; 946 } 947 948 int 949 ice_fdir_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 950 { 951 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 952 struct ci_tx_queue *txq; 953 int err; 954 struct ice_vsi *vsi; 955 struct ice_hw *hw; 956 struct ice_aqc_add_tx_qgrp *txq_elem; 957 struct ice_tlan_ctx tx_ctx; 958 int buf_len; 959 960 PMD_INIT_FUNC_TRACE(); 961 962 txq = pf->fdir.txq; 963 if (!txq || !txq->q_set) { 964 PMD_DRV_LOG(ERR, "FDIR TX queue %u is not available or setup", 965 tx_queue_id); 966 return -EINVAL; 967 } 968 969 buf_len = ice_struct_size(txq_elem, txqs, 1); 970 txq_elem = ice_malloc(hw, buf_len); 971 if (!txq_elem) 972 return -ENOMEM; 973 974 vsi = txq->ice_vsi; 975 hw = ICE_VSI_TO_HW(vsi); 976 977 memset(&tx_ctx, 0, sizeof(tx_ctx)); 978 txq_elem->num_txqs = 1; 979 txq_elem->txqs[0].txq_id = rte_cpu_to_le_16(txq->reg_idx); 980 981 tx_ctx.base = txq->tx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT; 982 tx_ctx.qlen = txq->nb_tx_desc; 983 tx_ctx.pf_num = hw->pf_id; 984 tx_ctx.vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; 985 tx_ctx.src_vsi = vsi->vsi_id; 986 tx_ctx.port_num = hw->port_info->lport; 987 tx_ctx.tso_ena = 1; /* tso enable */ 988 tx_ctx.tso_qnum = txq->reg_idx; /* index for tso state structure */ 989 tx_ctx.legacy_int = 1; /* Legacy or Advanced Host Interface */ 990 991 ice_set_ctx(hw, (uint8_t *)&tx_ctx, txq_elem->txqs[0].txq_ctx, 992 ice_tlan_ctx_info); 993 994 txq->qtx_tail = hw->hw_addr + QTX_COMM_DBELL(txq->reg_idx); 995 996 /* Init the Tx tail register*/ 997 ICE_PCI_REG_WRITE(txq->qtx_tail, 0); 998 999 /* Fix me, we assume TC always 0 here */ 1000 err = ice_ena_vsi_txq(hw->port_info, vsi->idx, 0, tx_queue_id, 1, 1001 txq_elem, buf_len, NULL); 1002 if (err) { 1003 PMD_DRV_LOG(ERR, "Failed to add FDIR txq"); 1004 rte_free(txq_elem); 1005 return -EIO; 1006 } 1007 /* store the schedule node id */ 1008 txq->q_teid = txq_elem->txqs[0].q_teid; 1009 1010 rte_free(txq_elem); 1011 return 0; 1012 } 1013 1014 static void 1015 ice_reset_tx_queue(struct ci_tx_queue *txq) 1016 { 1017 struct ci_tx_entry *txe; 1018 uint16_t i, prev, size; 1019 1020 if (!txq) { 1021 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL"); 1022 return; 1023 } 1024 1025 txe = txq->sw_ring; 1026 size = sizeof(struct ice_tx_desc) * txq->nb_tx_desc; 1027 for (i = 0; i < size; i++) 1028 ((volatile char *)txq->ice_tx_ring)[i] = 0; 1029 1030 prev = (uint16_t)(txq->nb_tx_desc - 1); 1031 for (i = 0; i < txq->nb_tx_desc; i++) { 1032 volatile struct ice_tx_desc *txd = &txq->ice_tx_ring[i]; 1033 1034 txd->cmd_type_offset_bsz = 1035 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE); 1036 txe[i].mbuf = NULL; 1037 txe[i].last_id = i; 1038 txe[prev].next_id = i; 1039 prev = i; 1040 } 1041 1042 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 1043 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 1044 1045 txq->tx_tail = 0; 1046 txq->nb_tx_used = 0; 1047 1048 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 1049 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 1050 } 1051 1052 int 1053 ice_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1054 { 1055 struct ci_tx_queue *txq; 1056 struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1057 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1058 struct ice_vsi *vsi = pf->main_vsi; 1059 uint16_t q_ids[1]; 1060 uint32_t q_teids[1]; 1061 uint16_t q_handle = tx_queue_id; 1062 int status; 1063 1064 if (tx_queue_id >= dev->data->nb_tx_queues) { 1065 PMD_DRV_LOG(ERR, "TX queue %u is out of range %u", 1066 tx_queue_id, dev->data->nb_tx_queues); 1067 return -EINVAL; 1068 } 1069 1070 txq = dev->data->tx_queues[tx_queue_id]; 1071 if (!txq) { 1072 PMD_DRV_LOG(ERR, "TX queue %u is not available", 1073 tx_queue_id); 1074 return -EINVAL; 1075 } 1076 1077 if (dev->data->tx_queue_state[tx_queue_id] == 1078 RTE_ETH_QUEUE_STATE_STOPPED) 1079 return 0; 1080 1081 q_ids[0] = txq->reg_idx; 1082 q_teids[0] = txq->q_teid; 1083 1084 /* Fix me, we assume TC always 0 here */ 1085 status = ice_dis_vsi_txq(hw->port_info, vsi->idx, 0, 1, &q_handle, 1086 q_ids, q_teids, ICE_NO_RESET, 0, NULL); 1087 if (status != ICE_SUCCESS) { 1088 PMD_DRV_LOG(DEBUG, "Failed to disable Lan Tx queue"); 1089 return -EINVAL; 1090 } 1091 1092 ci_txq_release_all_mbufs(txq, false); 1093 ice_reset_tx_queue(txq); 1094 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1095 1096 return 0; 1097 } 1098 1099 int 1100 ice_fdir_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1101 { 1102 struct ice_rx_queue *rxq; 1103 int err; 1104 struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1105 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1106 1107 rxq = pf->fdir.rxq; 1108 1109 err = ice_switch_rx_queue(hw, rxq->reg_idx, false); 1110 if (err) { 1111 PMD_DRV_LOG(ERR, "Failed to switch FDIR RX queue %u off", 1112 rx_queue_id); 1113 return -EINVAL; 1114 } 1115 rxq->rx_rel_mbufs(rxq); 1116 1117 return 0; 1118 } 1119 1120 int 1121 ice_fdir_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1122 { 1123 struct ci_tx_queue *txq; 1124 struct ice_hw *hw = ICE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1125 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1126 struct ice_vsi *vsi = pf->main_vsi; 1127 uint16_t q_ids[1]; 1128 uint32_t q_teids[1]; 1129 uint16_t q_handle = tx_queue_id; 1130 int status; 1131 1132 txq = pf->fdir.txq; 1133 if (!txq) { 1134 PMD_DRV_LOG(ERR, "TX queue %u is not available", 1135 tx_queue_id); 1136 return -EINVAL; 1137 } 1138 if (txq->qtx_tail == NULL) { 1139 PMD_DRV_LOG(INFO, "TX queue %u not started", tx_queue_id); 1140 return 0; 1141 } 1142 vsi = txq->ice_vsi; 1143 1144 q_ids[0] = txq->reg_idx; 1145 q_teids[0] = txq->q_teid; 1146 1147 /* Fix me, we assume TC always 0 here */ 1148 status = ice_dis_vsi_txq(hw->port_info, vsi->idx, 0, 1, &q_handle, 1149 q_ids, q_teids, ICE_NO_RESET, 0, NULL); 1150 if (status != ICE_SUCCESS) { 1151 PMD_DRV_LOG(DEBUG, "Failed to disable Lan Tx queue"); 1152 return -EINVAL; 1153 } 1154 1155 ci_txq_release_all_mbufs(txq, false); 1156 txq->qtx_tail = NULL; 1157 1158 return 0; 1159 } 1160 1161 int 1162 ice_rx_queue_setup(struct rte_eth_dev *dev, 1163 uint16_t queue_idx, 1164 uint16_t nb_desc, 1165 unsigned int socket_id, 1166 const struct rte_eth_rxconf *rx_conf, 1167 struct rte_mempool *mp) 1168 { 1169 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1170 struct ice_adapter *ad = 1171 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1172 struct ice_vsi *vsi = pf->main_vsi; 1173 struct ice_rx_queue *rxq; 1174 const struct rte_memzone *rz; 1175 uint32_t ring_size; 1176 uint16_t len; 1177 int use_def_burst_func = 1; 1178 uint64_t offloads; 1179 uint16_t n_seg = rx_conf->rx_nseg; 1180 uint16_t i; 1181 1182 if (nb_desc % ICE_ALIGN_RING_DESC != 0 || 1183 nb_desc > ICE_MAX_RING_DESC || 1184 nb_desc < ICE_MIN_RING_DESC) { 1185 PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is " 1186 "invalid", nb_desc); 1187 return -EINVAL; 1188 } 1189 1190 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 1191 1192 if (mp) 1193 n_seg = 1; 1194 1195 if (n_seg > 1 && !(offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 1196 PMD_INIT_LOG(ERR, "port %u queue index %u split offload not configured", 1197 dev->data->port_id, queue_idx); 1198 return -EINVAL; 1199 } 1200 1201 /* Free memory if needed */ 1202 if (dev->data->rx_queues[queue_idx]) { 1203 ice_rx_queue_release(dev->data->rx_queues[queue_idx]); 1204 dev->data->rx_queues[queue_idx] = NULL; 1205 } 1206 1207 /* Allocate the rx queue data structure */ 1208 rxq = rte_zmalloc_socket(NULL, 1209 sizeof(struct ice_rx_queue), 1210 RTE_CACHE_LINE_SIZE, 1211 socket_id); 1212 1213 if (!rxq) { 1214 PMD_INIT_LOG(ERR, "Failed to allocate memory for " 1215 "rx queue data structure"); 1216 return -ENOMEM; 1217 } 1218 1219 rxq->rxseg_nb = n_seg; 1220 if (n_seg > 1) { 1221 for (i = 0; i < n_seg; i++) 1222 memcpy(&rxq->rxseg[i], &rx_conf->rx_seg[i].split, 1223 sizeof(struct rte_eth_rxseg_split)); 1224 1225 rxq->mp = rxq->rxseg[0].mp; 1226 } else { 1227 rxq->mp = mp; 1228 } 1229 1230 rxq->nb_rx_desc = nb_desc; 1231 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 1232 rxq->queue_id = queue_idx; 1233 rxq->offloads = offloads; 1234 1235 rxq->reg_idx = vsi->base_queue + queue_idx; 1236 rxq->port_id = dev->data->port_id; 1237 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 1238 rxq->crc_len = RTE_ETHER_CRC_LEN; 1239 else 1240 rxq->crc_len = 0; 1241 1242 rxq->drop_en = rx_conf->rx_drop_en; 1243 rxq->vsi = vsi; 1244 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 1245 rxq->proto_xtr = pf->proto_xtr != NULL ? 1246 pf->proto_xtr[queue_idx] : PROTO_XTR_NONE; 1247 if (rxq->proto_xtr != PROTO_XTR_NONE && 1248 ad->devargs.xtr_flag_offs[rxq->proto_xtr] != 0xff) 1249 rxq->xtr_ol_flag = 1ULL << ad->devargs.xtr_flag_offs[rxq->proto_xtr]; 1250 rxq->xtr_field_offs = ad->devargs.xtr_field_offs; 1251 1252 /* Allocate the maximum number of RX ring hardware descriptor. */ 1253 len = ICE_MAX_RING_DESC; 1254 1255 /** 1256 * Allocating a little more memory because vectorized/bulk_alloc Rx 1257 * functions doesn't check boundaries each time. 1258 */ 1259 len += ICE_RX_MAX_BURST; 1260 1261 /* Allocate the maximum number of RX ring hardware descriptor. */ 1262 ring_size = sizeof(union ice_rx_flex_desc) * len; 1263 ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN); 1264 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 1265 ring_size, ICE_RING_BASE_ALIGN, 1266 socket_id); 1267 if (!rz) { 1268 ice_rx_queue_release(rxq); 1269 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX"); 1270 return -ENOMEM; 1271 } 1272 1273 rxq->mz = rz; 1274 /* Zero all the descriptors in the ring. */ 1275 memset(rz->addr, 0, ring_size); 1276 1277 rxq->rx_ring_dma = rz->iova; 1278 rxq->rx_ring = rz->addr; 1279 1280 /* always reserve more for bulk alloc */ 1281 len = (uint16_t)(nb_desc + ICE_RX_MAX_BURST); 1282 1283 /* Allocate the software ring. */ 1284 rxq->sw_ring = rte_zmalloc_socket(NULL, 1285 sizeof(struct ice_rx_entry) * len, 1286 RTE_CACHE_LINE_SIZE, 1287 socket_id); 1288 if (!rxq->sw_ring) { 1289 ice_rx_queue_release(rxq); 1290 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring"); 1291 return -ENOMEM; 1292 } 1293 1294 ice_reset_rx_queue(rxq); 1295 rxq->q_set = true; 1296 dev->data->rx_queues[queue_idx] = rxq; 1297 rxq->rx_rel_mbufs = _ice_rx_queue_release_mbufs; 1298 1299 use_def_burst_func = ice_check_rx_burst_bulk_alloc_preconditions(rxq); 1300 1301 if (!use_def_burst_func) { 1302 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 1303 "satisfied. Rx Burst Bulk Alloc function will be " 1304 "used on port=%d, queue=%d.", 1305 rxq->port_id, rxq->queue_id); 1306 } else { 1307 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 1308 "not satisfied, Scattered Rx is requested. " 1309 "on port=%d, queue=%d.", 1310 rxq->port_id, rxq->queue_id); 1311 ad->rx_bulk_alloc_allowed = false; 1312 } 1313 1314 return 0; 1315 } 1316 1317 void 1318 ice_rx_queue_release(void *rxq) 1319 { 1320 struct ice_rx_queue *q = (struct ice_rx_queue *)rxq; 1321 1322 if (!q) { 1323 PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL"); 1324 return; 1325 } 1326 1327 if (q->rx_rel_mbufs != NULL) 1328 q->rx_rel_mbufs(q); 1329 rte_free(q->sw_ring); 1330 rte_memzone_free(q->mz); 1331 rte_free(q); 1332 } 1333 1334 int 1335 ice_tx_queue_setup(struct rte_eth_dev *dev, 1336 uint16_t queue_idx, 1337 uint16_t nb_desc, 1338 unsigned int socket_id, 1339 const struct rte_eth_txconf *tx_conf) 1340 { 1341 struct ice_pf *pf = ICE_DEV_PRIVATE_TO_PF(dev->data->dev_private); 1342 struct ice_vsi *vsi = pf->main_vsi; 1343 struct ci_tx_queue *txq; 1344 const struct rte_memzone *tz; 1345 uint32_t ring_size; 1346 uint16_t tx_rs_thresh, tx_free_thresh; 1347 uint64_t offloads; 1348 1349 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1350 1351 if (nb_desc % ICE_ALIGN_RING_DESC != 0 || 1352 nb_desc > ICE_MAX_RING_DESC || 1353 nb_desc < ICE_MIN_RING_DESC) { 1354 PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is " 1355 "invalid", nb_desc); 1356 return -EINVAL; 1357 } 1358 1359 /** 1360 * The following two parameters control the setting of the RS bit on 1361 * transmit descriptors. TX descriptors will have their RS bit set 1362 * after txq->tx_rs_thresh descriptors have been used. The TX 1363 * descriptor ring will be cleaned after txq->tx_free_thresh 1364 * descriptors are used or if the number of descriptors required to 1365 * transmit a packet is greater than the number of free TX descriptors. 1366 * 1367 * The following constraints must be satisfied: 1368 * - tx_rs_thresh must be greater than 0. 1369 * - tx_rs_thresh must be less than the size of the ring minus 2. 1370 * - tx_rs_thresh must be less than or equal to tx_free_thresh. 1371 * - tx_rs_thresh must be a divisor of the ring size. 1372 * - tx_free_thresh must be greater than 0. 1373 * - tx_free_thresh must be less than the size of the ring minus 3. 1374 * - tx_free_thresh + tx_rs_thresh must not exceed nb_desc. 1375 * 1376 * One descriptor in the TX ring is used as a sentinel to avoid a H/W 1377 * race condition, hence the maximum threshold constraints. When set 1378 * to zero use default values. 1379 */ 1380 tx_free_thresh = (uint16_t)(tx_conf->tx_free_thresh ? 1381 tx_conf->tx_free_thresh : 1382 ICE_DEFAULT_TX_FREE_THRESH); 1383 /* force tx_rs_thresh to adapt an aggressive tx_free_thresh */ 1384 tx_rs_thresh = 1385 (ICE_DEFAULT_TX_RSBIT_THRESH + tx_free_thresh > nb_desc) ? 1386 nb_desc - tx_free_thresh : ICE_DEFAULT_TX_RSBIT_THRESH; 1387 if (tx_conf->tx_rs_thresh) 1388 tx_rs_thresh = tx_conf->tx_rs_thresh; 1389 if (tx_rs_thresh + tx_free_thresh > nb_desc) { 1390 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not " 1391 "exceed nb_desc. (tx_rs_thresh=%u " 1392 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)", 1393 (unsigned int)tx_rs_thresh, 1394 (unsigned int)tx_free_thresh, 1395 (unsigned int)nb_desc, 1396 (int)dev->data->port_id, 1397 (int)queue_idx); 1398 return -EINVAL; 1399 } 1400 if (tx_rs_thresh >= (nb_desc - 2)) { 1401 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " 1402 "number of TX descriptors minus 2. " 1403 "(tx_rs_thresh=%u port=%d queue=%d)", 1404 (unsigned int)tx_rs_thresh, 1405 (int)dev->data->port_id, 1406 (int)queue_idx); 1407 return -EINVAL; 1408 } 1409 if (tx_free_thresh >= (nb_desc - 3)) { 1410 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " 1411 "tx_free_thresh must be less than the " 1412 "number of TX descriptors minus 3. " 1413 "(tx_free_thresh=%u port=%d queue=%d)", 1414 (unsigned int)tx_free_thresh, 1415 (int)dev->data->port_id, 1416 (int)queue_idx); 1417 return -EINVAL; 1418 } 1419 if (tx_rs_thresh > tx_free_thresh) { 1420 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or " 1421 "equal to tx_free_thresh. (tx_free_thresh=%u" 1422 " tx_rs_thresh=%u port=%d queue=%d)", 1423 (unsigned int)tx_free_thresh, 1424 (unsigned int)tx_rs_thresh, 1425 (int)dev->data->port_id, 1426 (int)queue_idx); 1427 return -EINVAL; 1428 } 1429 if ((nb_desc % tx_rs_thresh) != 0) { 1430 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the " 1431 "number of TX descriptors. (tx_rs_thresh=%u" 1432 " port=%d queue=%d)", 1433 (unsigned int)tx_rs_thresh, 1434 (int)dev->data->port_id, 1435 (int)queue_idx); 1436 return -EINVAL; 1437 } 1438 if (tx_rs_thresh > 1 && tx_conf->tx_thresh.wthresh != 0) { 1439 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if " 1440 "tx_rs_thresh is greater than 1. " 1441 "(tx_rs_thresh=%u port=%d queue=%d)", 1442 (unsigned int)tx_rs_thresh, 1443 (int)dev->data->port_id, 1444 (int)queue_idx); 1445 return -EINVAL; 1446 } 1447 1448 /* Free memory if needed. */ 1449 if (dev->data->tx_queues[queue_idx]) { 1450 ice_tx_queue_release(dev->data->tx_queues[queue_idx]); 1451 dev->data->tx_queues[queue_idx] = NULL; 1452 } 1453 1454 /* Allocate the TX queue data structure. */ 1455 txq = rte_zmalloc_socket(NULL, 1456 sizeof(struct ci_tx_queue), 1457 RTE_CACHE_LINE_SIZE, 1458 socket_id); 1459 if (!txq) { 1460 PMD_INIT_LOG(ERR, "Failed to allocate memory for " 1461 "tx queue structure"); 1462 return -ENOMEM; 1463 } 1464 1465 /* Allocate TX hardware ring descriptors. */ 1466 ring_size = sizeof(struct ice_tx_desc) * ICE_MAX_RING_DESC; 1467 ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN); 1468 tz = rte_eth_dma_zone_reserve(dev, "ice_tx_ring", queue_idx, 1469 ring_size, ICE_RING_BASE_ALIGN, 1470 socket_id); 1471 if (!tz) { 1472 ice_tx_queue_release(txq); 1473 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX"); 1474 return -ENOMEM; 1475 } 1476 1477 txq->mz = tz; 1478 txq->nb_tx_desc = nb_desc; 1479 txq->tx_rs_thresh = tx_rs_thresh; 1480 txq->tx_free_thresh = tx_free_thresh; 1481 txq->queue_id = queue_idx; 1482 1483 txq->reg_idx = vsi->base_queue + queue_idx; 1484 txq->port_id = dev->data->port_id; 1485 txq->offloads = offloads; 1486 txq->ice_vsi = vsi; 1487 txq->tx_deferred_start = tx_conf->tx_deferred_start; 1488 1489 txq->tx_ring_dma = tz->iova; 1490 txq->ice_tx_ring = tz->addr; 1491 1492 /* Allocate software ring */ 1493 txq->sw_ring = 1494 rte_zmalloc_socket(NULL, 1495 sizeof(struct ci_tx_entry) * nb_desc, 1496 RTE_CACHE_LINE_SIZE, 1497 socket_id); 1498 if (!txq->sw_ring) { 1499 ice_tx_queue_release(txq); 1500 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring"); 1501 return -ENOMEM; 1502 } 1503 1504 ice_reset_tx_queue(txq); 1505 txq->q_set = true; 1506 dev->data->tx_queues[queue_idx] = txq; 1507 ice_set_tx_function_flag(dev, txq); 1508 1509 return 0; 1510 } 1511 1512 void 1513 ice_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1514 { 1515 ice_rx_queue_release(dev->data->rx_queues[qid]); 1516 } 1517 1518 void 1519 ice_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1520 { 1521 ice_tx_queue_release(dev->data->tx_queues[qid]); 1522 } 1523 1524 void 1525 ice_tx_queue_release(void *txq) 1526 { 1527 struct ci_tx_queue *q = (struct ci_tx_queue *)txq; 1528 1529 if (!q) { 1530 PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL"); 1531 return; 1532 } 1533 1534 ci_txq_release_all_mbufs(q, false); 1535 rte_free(q->sw_ring); 1536 rte_memzone_free(q->mz); 1537 rte_free(q); 1538 } 1539 1540 void 1541 ice_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 1542 struct rte_eth_rxq_info *qinfo) 1543 { 1544 struct ice_rx_queue *rxq; 1545 1546 rxq = dev->data->rx_queues[queue_id]; 1547 1548 qinfo->mp = rxq->mp; 1549 qinfo->scattered_rx = dev->data->scattered_rx; 1550 qinfo->nb_desc = rxq->nb_rx_desc; 1551 1552 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 1553 qinfo->conf.rx_drop_en = rxq->drop_en; 1554 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 1555 } 1556 1557 void 1558 ice_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 1559 struct rte_eth_txq_info *qinfo) 1560 { 1561 struct ci_tx_queue *txq; 1562 1563 txq = dev->data->tx_queues[queue_id]; 1564 1565 qinfo->nb_desc = txq->nb_tx_desc; 1566 1567 qinfo->conf.tx_thresh.pthresh = ICE_DEFAULT_TX_PTHRESH; 1568 qinfo->conf.tx_thresh.hthresh = ICE_DEFAULT_TX_HTHRESH; 1569 qinfo->conf.tx_thresh.wthresh = ICE_DEFAULT_TX_WTHRESH; 1570 1571 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 1572 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; 1573 qinfo->conf.offloads = txq->offloads; 1574 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 1575 } 1576 1577 uint32_t 1578 ice_rx_queue_count(void *rx_queue) 1579 { 1580 #define ICE_RXQ_SCAN_INTERVAL 4 1581 volatile union ice_rx_flex_desc *rxdp; 1582 struct ice_rx_queue *rxq; 1583 uint16_t desc = 0; 1584 1585 rxq = rx_queue; 1586 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1587 while ((desc < rxq->nb_rx_desc) && 1588 rte_le_to_cpu_16(rxdp->wb.status_error0) & 1589 (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)) { 1590 /** 1591 * Check the DD bit of a rx descriptor of each 4 in a group, 1592 * to avoid checking too frequently and downgrading performance 1593 * too much. 1594 */ 1595 desc += ICE_RXQ_SCAN_INTERVAL; 1596 rxdp += ICE_RXQ_SCAN_INTERVAL; 1597 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 1598 rxdp = &(rxq->rx_ring[rxq->rx_tail + 1599 desc - rxq->nb_rx_desc]); 1600 } 1601 1602 return desc; 1603 } 1604 1605 #define ICE_RX_FLEX_ERR0_BITS \ 1606 ((1 << ICE_RX_FLEX_DESC_STATUS0_HBO_S) | \ 1607 (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | \ 1608 (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) | \ 1609 (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S) | \ 1610 (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S) | \ 1611 (1 << ICE_RX_FLEX_DESC_STATUS0_RXE_S)) 1612 1613 /* Rx L3/L4 checksum */ 1614 static inline uint64_t 1615 ice_rxd_error_to_pkt_flags(uint16_t stat_err0) 1616 { 1617 uint64_t flags = 0; 1618 1619 /* check if HW has decoded the packet and checksum */ 1620 if (unlikely(!(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_L3L4P_S)))) 1621 return 0; 1622 1623 if (likely(!(stat_err0 & ICE_RX_FLEX_ERR0_BITS))) { 1624 flags |= (RTE_MBUF_F_RX_IP_CKSUM_GOOD | 1625 RTE_MBUF_F_RX_L4_CKSUM_GOOD | 1626 RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD); 1627 return flags; 1628 } 1629 1630 if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S))) 1631 flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 1632 else 1633 flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 1634 1635 if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S))) 1636 flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 1637 else 1638 flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 1639 1640 if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S))) 1641 flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 1642 1643 if (unlikely(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S))) 1644 flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_BAD; 1645 else 1646 flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD; 1647 1648 return flags; 1649 } 1650 1651 static inline void 1652 ice_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union ice_rx_flex_desc *rxdp) 1653 { 1654 if (rte_le_to_cpu_16(rxdp->wb.status_error0) & 1655 (1 << ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) { 1656 mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 1657 mb->vlan_tci = 1658 rte_le_to_cpu_16(rxdp->wb.l2tag1); 1659 PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u", 1660 rte_le_to_cpu_16(rxdp->wb.l2tag1)); 1661 } else { 1662 mb->vlan_tci = 0; 1663 } 1664 1665 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 1666 if (rte_le_to_cpu_16(rxdp->wb.status_error1) & 1667 (1 << ICE_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) { 1668 mb->ol_flags |= RTE_MBUF_F_RX_QINQ_STRIPPED | RTE_MBUF_F_RX_QINQ | 1669 RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN; 1670 mb->vlan_tci_outer = mb->vlan_tci; 1671 mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd); 1672 PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u", 1673 rte_le_to_cpu_16(rxdp->wb.l2tag2_1st), 1674 rte_le_to_cpu_16(rxdp->wb.l2tag2_2nd)); 1675 } else { 1676 mb->vlan_tci_outer = 0; 1677 } 1678 #endif 1679 PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u", 1680 mb->vlan_tci, mb->vlan_tci_outer); 1681 } 1682 1683 #define ICE_LOOK_AHEAD 8 1684 #if (ICE_LOOK_AHEAD != 8) 1685 #error "PMD ICE: ICE_LOOK_AHEAD must be 8\n" 1686 #endif 1687 1688 #define ICE_PTP_TS_VALID 0x1 1689 1690 static inline int 1691 ice_rx_scan_hw_ring(struct ice_rx_queue *rxq) 1692 { 1693 volatile union ice_rx_flex_desc *rxdp; 1694 struct ice_rx_entry *rxep; 1695 struct rte_mbuf *mb; 1696 uint16_t stat_err0; 1697 uint16_t pkt_len, hdr_len; 1698 int32_t s[ICE_LOOK_AHEAD], nb_dd; 1699 int32_t i, j, nb_rx = 0; 1700 uint64_t pkt_flags = 0; 1701 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 1702 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 1703 bool is_tsinit = false; 1704 uint64_t ts_ns; 1705 struct ice_vsi *vsi = rxq->vsi; 1706 struct ice_hw *hw = ICE_VSI_TO_HW(vsi); 1707 struct ice_adapter *ad = rxq->vsi->adapter; 1708 #endif 1709 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1710 rxep = &rxq->sw_ring[rxq->rx_tail]; 1711 1712 stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); 1713 1714 /* Make sure there is at least 1 packet to receive */ 1715 if (!(stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S))) 1716 return 0; 1717 1718 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 1719 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 1720 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 1721 1722 if (unlikely(sw_cur_time - rxq->hw_time_update > 4)) 1723 is_tsinit = 1; 1724 } 1725 #endif 1726 1727 /** 1728 * Scan LOOK_AHEAD descriptors at a time to determine which 1729 * descriptors reference packets that are ready to be received. 1730 */ 1731 for (i = 0; i < ICE_RX_MAX_BURST; i += ICE_LOOK_AHEAD, 1732 rxdp += ICE_LOOK_AHEAD, rxep += ICE_LOOK_AHEAD) { 1733 /* Read desc statuses backwards to avoid race condition */ 1734 for (j = ICE_LOOK_AHEAD - 1; j >= 0; j--) 1735 s[j] = rte_le_to_cpu_16(rxdp[j].wb.status_error0); 1736 1737 rte_smp_rmb(); 1738 1739 /* Compute how many status bits were set */ 1740 for (j = 0, nb_dd = 0; j < ICE_LOOK_AHEAD; j++) 1741 nb_dd += s[j] & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S); 1742 1743 nb_rx += nb_dd; 1744 1745 /* Translate descriptor info to mbuf parameters */ 1746 for (j = 0; j < nb_dd; j++) { 1747 mb = rxep[j].mbuf; 1748 pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) & 1749 ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 1750 mb->data_len = pkt_len; 1751 mb->pkt_len = pkt_len; 1752 1753 if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 1754 pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) & 1755 ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 1756 mb->data_len = pkt_len; 1757 mb->pkt_len = pkt_len; 1758 } else { 1759 mb->nb_segs = (uint16_t)(mb->nb_segs + mb->next->nb_segs); 1760 mb->next->next = NULL; 1761 hdr_len = rte_le_to_cpu_16(rxdp[j].wb.hdr_len_sph_flex_flags1) & 1762 ICE_RX_FLEX_DESC_HEADER_LEN_M; 1763 pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) & 1764 ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 1765 mb->data_len = hdr_len; 1766 mb->pkt_len = hdr_len + pkt_len; 1767 mb->next->data_len = pkt_len; 1768 #ifdef RTE_ETHDEV_DEBUG_RX 1769 rte_pktmbuf_dump(stdout, mb, rte_pktmbuf_pkt_len(mb)); 1770 #endif 1771 } 1772 1773 mb->ol_flags = 0; 1774 stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0); 1775 pkt_flags = ice_rxd_error_to_pkt_flags(stat_err0); 1776 mb->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M & 1777 rte_le_to_cpu_16(rxdp[j].wb.ptype_flex_flags0)]; 1778 ice_rxd_to_vlan_tci(mb, &rxdp[j]); 1779 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, mb, &rxdp[j]); 1780 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 1781 if (ice_timestamp_dynflag > 0 && 1782 (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { 1783 rxq->time_high = 1784 rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high); 1785 if (unlikely(is_tsinit)) { 1786 ts_ns = ice_tstamp_convert_32b_64b(hw, ad, 1, 1787 rxq->time_high); 1788 rxq->hw_time_low = (uint32_t)ts_ns; 1789 rxq->hw_time_high = (uint32_t)(ts_ns >> 32); 1790 is_tsinit = false; 1791 } else { 1792 if (rxq->time_high < rxq->hw_time_low) 1793 rxq->hw_time_high += 1; 1794 ts_ns = (uint64_t)rxq->hw_time_high << 32 | rxq->time_high; 1795 rxq->hw_time_low = rxq->time_high; 1796 } 1797 rxq->hw_time_update = rte_get_timer_cycles() / 1798 (rte_get_timer_hz() / 1000); 1799 *RTE_MBUF_DYNFIELD(mb, 1800 ice_timestamp_dynfield_offset, 1801 rte_mbuf_timestamp_t *) = ts_ns; 1802 pkt_flags |= ice_timestamp_dynflag; 1803 } 1804 1805 if (ad->ptp_ena && ((mb->packet_type & 1806 RTE_PTYPE_L2_MASK) == RTE_PTYPE_L2_ETHER_TIMESYNC)) { 1807 rxq->time_high = 1808 rte_le_to_cpu_32(rxdp[j].wb.flex_ts.ts_high); 1809 mb->timesync = rxq->queue_id; 1810 pkt_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; 1811 if (rxdp[j].wb.time_stamp_low & 1812 ICE_PTP_TS_VALID) 1813 pkt_flags |= 1814 RTE_MBUF_F_RX_IEEE1588_TMST; 1815 } 1816 #endif 1817 mb->ol_flags |= pkt_flags; 1818 } 1819 1820 for (j = 0; j < ICE_LOOK_AHEAD; j++) 1821 rxq->rx_stage[i + j] = rxep[j].mbuf; 1822 1823 if (nb_dd != ICE_LOOK_AHEAD) 1824 break; 1825 } 1826 1827 /* Clear software ring entries */ 1828 for (i = 0; i < nb_rx; i++) 1829 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1830 1831 PMD_RX_LOG(DEBUG, "ice_rx_scan_hw_ring: " 1832 "port_id=%u, queue_id=%u, nb_rx=%d", 1833 rxq->port_id, rxq->queue_id, nb_rx); 1834 1835 return nb_rx; 1836 } 1837 1838 static inline uint16_t 1839 ice_rx_fill_from_stage(struct ice_rx_queue *rxq, 1840 struct rte_mbuf **rx_pkts, 1841 uint16_t nb_pkts) 1842 { 1843 uint16_t i; 1844 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1845 1846 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1847 1848 for (i = 0; i < nb_pkts; i++) 1849 rx_pkts[i] = stage[i]; 1850 1851 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1852 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1853 1854 return nb_pkts; 1855 } 1856 1857 static inline int 1858 ice_rx_alloc_bufs(struct ice_rx_queue *rxq) 1859 { 1860 volatile union ice_rx_flex_desc *rxdp; 1861 struct ice_rx_entry *rxep; 1862 struct rte_mbuf *mb; 1863 uint16_t alloc_idx, i; 1864 uint64_t dma_addr; 1865 int diag, diag_pay; 1866 uint64_t pay_addr; 1867 struct rte_mbuf *mbufs_pay[rxq->rx_free_thresh]; 1868 1869 /* Allocate buffers in bulk */ 1870 alloc_idx = (uint16_t)(rxq->rx_free_trigger - 1871 (rxq->rx_free_thresh - 1)); 1872 rxep = &rxq->sw_ring[alloc_idx]; 1873 diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep, 1874 rxq->rx_free_thresh); 1875 if (unlikely(diag != 0)) { 1876 PMD_RX_LOG(ERR, "Failed to get mbufs in bulk"); 1877 return -ENOMEM; 1878 } 1879 1880 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) { 1881 diag_pay = rte_mempool_get_bulk(rxq->rxseg[1].mp, 1882 (void *)mbufs_pay, rxq->rx_free_thresh); 1883 if (unlikely(diag_pay != 0)) { 1884 rte_mempool_put_bulk(rxq->mp, (void *)rxep, 1885 rxq->rx_free_thresh); 1886 PMD_RX_LOG(ERR, "Failed to get payload mbufs in bulk"); 1887 return -ENOMEM; 1888 } 1889 } 1890 1891 rxdp = &rxq->rx_ring[alloc_idx]; 1892 for (i = 0; i < rxq->rx_free_thresh; i++) { 1893 if (likely(i < (rxq->rx_free_thresh - 1))) 1894 /* Prefetch next mbuf */ 1895 rte_prefetch0(rxep[i + 1].mbuf); 1896 1897 mb = rxep[i].mbuf; 1898 rte_mbuf_refcnt_set(mb, 1); 1899 mb->data_off = RTE_PKTMBUF_HEADROOM; 1900 mb->nb_segs = 1; 1901 mb->port = rxq->port_id; 1902 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1903 1904 if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 1905 mb->next = NULL; 1906 rxdp[i].read.hdr_addr = 0; 1907 rxdp[i].read.pkt_addr = dma_addr; 1908 } else { 1909 mb->next = mbufs_pay[i]; 1910 pay_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbufs_pay[i])); 1911 rxdp[i].read.hdr_addr = dma_addr; 1912 rxdp[i].read.pkt_addr = pay_addr; 1913 } 1914 } 1915 1916 /* Update Rx tail register */ 1917 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger); 1918 1919 rxq->rx_free_trigger = 1920 (uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh); 1921 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1922 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 1923 1924 return 0; 1925 } 1926 1927 static inline uint16_t 1928 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1929 { 1930 struct ice_rx_queue *rxq = (struct ice_rx_queue *)rx_queue; 1931 uint16_t nb_rx = 0; 1932 1933 if (!nb_pkts) 1934 return 0; 1935 1936 if (rxq->rx_nb_avail) 1937 return ice_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1938 1939 nb_rx = (uint16_t)ice_rx_scan_hw_ring(rxq); 1940 rxq->rx_next_avail = 0; 1941 rxq->rx_nb_avail = nb_rx; 1942 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1943 1944 if (rxq->rx_tail > rxq->rx_free_trigger) { 1945 if (ice_rx_alloc_bufs(rxq) != 0) { 1946 uint16_t i, j; 1947 1948 rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed += 1949 rxq->rx_free_thresh; 1950 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed for " 1951 "port_id=%u, queue_id=%u", 1952 rxq->port_id, rxq->queue_id); 1953 rxq->rx_nb_avail = 0; 1954 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1955 for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++) 1956 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1957 1958 return 0; 1959 } 1960 } 1961 1962 if (rxq->rx_tail >= rxq->nb_rx_desc) 1963 rxq->rx_tail = 0; 1964 1965 if (rxq->rx_nb_avail) 1966 return ice_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1967 1968 return 0; 1969 } 1970 1971 static uint16_t 1972 ice_recv_pkts_bulk_alloc(void *rx_queue, 1973 struct rte_mbuf **rx_pkts, 1974 uint16_t nb_pkts) 1975 { 1976 uint16_t nb_rx = 0; 1977 uint16_t n; 1978 uint16_t count; 1979 1980 if (unlikely(nb_pkts == 0)) 1981 return nb_rx; 1982 1983 if (likely(nb_pkts <= ICE_RX_MAX_BURST)) 1984 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1985 1986 while (nb_pkts) { 1987 n = RTE_MIN(nb_pkts, ICE_RX_MAX_BURST); 1988 count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1989 nb_rx = (uint16_t)(nb_rx + count); 1990 nb_pkts = (uint16_t)(nb_pkts - count); 1991 if (count < n) 1992 break; 1993 } 1994 1995 return nb_rx; 1996 } 1997 1998 static uint16_t 1999 ice_recv_scattered_pkts(void *rx_queue, 2000 struct rte_mbuf **rx_pkts, 2001 uint16_t nb_pkts) 2002 { 2003 struct ice_rx_queue *rxq = rx_queue; 2004 volatile union ice_rx_flex_desc *rx_ring = rxq->rx_ring; 2005 volatile union ice_rx_flex_desc *rxdp; 2006 union ice_rx_flex_desc rxd; 2007 struct ice_rx_entry *sw_ring = rxq->sw_ring; 2008 struct ice_rx_entry *rxe; 2009 struct rte_mbuf *first_seg = rxq->pkt_first_seg; 2010 struct rte_mbuf *last_seg = rxq->pkt_last_seg; 2011 struct rte_mbuf *nmb; /* new allocated mbuf */ 2012 struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */ 2013 uint16_t rx_id = rxq->rx_tail; 2014 uint16_t nb_rx = 0; 2015 uint16_t nb_hold = 0; 2016 uint16_t rx_packet_len; 2017 uint16_t rx_stat_err0; 2018 uint64_t dma_addr; 2019 uint64_t pkt_flags; 2020 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 2021 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 2022 bool is_tsinit = false; 2023 uint64_t ts_ns; 2024 struct ice_vsi *vsi = rxq->vsi; 2025 struct ice_hw *hw = ICE_VSI_TO_HW(vsi); 2026 struct ice_adapter *ad = rxq->vsi->adapter; 2027 2028 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 2029 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 2030 2031 if (unlikely(sw_cur_time - rxq->hw_time_update > 4)) 2032 is_tsinit = true; 2033 } 2034 #endif 2035 2036 while (nb_rx < nb_pkts) { 2037 rxdp = &rx_ring[rx_id]; 2038 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); 2039 2040 /* Check the DD bit first */ 2041 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S))) 2042 break; 2043 2044 /* allocate mbuf */ 2045 nmb = rte_mbuf_raw_alloc(rxq->mp); 2046 if (unlikely(!nmb)) { 2047 rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++; 2048 break; 2049 } 2050 rxd = *rxdp; /* copy descriptor in ring to temp variable*/ 2051 2052 nb_hold++; 2053 rxe = &sw_ring[rx_id]; /* get corresponding mbuf in SW ring */ 2054 rx_id++; 2055 if (unlikely(rx_id == rxq->nb_rx_desc)) 2056 rx_id = 0; 2057 2058 /* Prefetch next mbuf */ 2059 rte_prefetch0(sw_ring[rx_id].mbuf); 2060 2061 /** 2062 * When next RX descriptor is on a cache line boundary, 2063 * prefetch the next 4 RX descriptors and next 8 pointers 2064 * to mbufs. 2065 */ 2066 if ((rx_id & 0x3) == 0) { 2067 rte_prefetch0(&rx_ring[rx_id]); 2068 rte_prefetch0(&sw_ring[rx_id]); 2069 } 2070 2071 rxm = rxe->mbuf; 2072 rxe->mbuf = nmb; 2073 dma_addr = 2074 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 2075 2076 /* Set data buffer address and data length of the mbuf */ 2077 rxdp->read.hdr_addr = 0; 2078 rxdp->read.pkt_addr = dma_addr; 2079 rx_packet_len = rte_le_to_cpu_16(rxd.wb.pkt_len) & 2080 ICE_RX_FLX_DESC_PKT_LEN_M; 2081 rxm->data_len = rx_packet_len; 2082 rxm->data_off = RTE_PKTMBUF_HEADROOM; 2083 2084 /** 2085 * If this is the first buffer of the received packet, set the 2086 * pointer to the first mbuf of the packet and initialize its 2087 * context. Otherwise, update the total length and the number 2088 * of segments of the current scattered packet, and update the 2089 * pointer to the last mbuf of the current packet. 2090 */ 2091 if (!first_seg) { 2092 first_seg = rxm; 2093 first_seg->nb_segs = 1; 2094 first_seg->pkt_len = rx_packet_len; 2095 } else { 2096 first_seg->pkt_len = 2097 (uint16_t)(first_seg->pkt_len + 2098 rx_packet_len); 2099 first_seg->nb_segs++; 2100 last_seg->next = rxm; 2101 } 2102 2103 /** 2104 * If this is not the last buffer of the received packet, 2105 * update the pointer to the last mbuf of the current scattered 2106 * packet and continue to parse the RX ring. 2107 */ 2108 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_EOF_S))) { 2109 last_seg = rxm; 2110 continue; 2111 } 2112 2113 /** 2114 * This is the last buffer of the received packet. If the CRC 2115 * is not stripped by the hardware: 2116 * - Subtract the CRC length from the total packet length. 2117 * - If the last buffer only contains the whole CRC or a part 2118 * of it, free the mbuf associated to the last buffer. If part 2119 * of the CRC is also contained in the previous mbuf, subtract 2120 * the length of that CRC part from the data length of the 2121 * previous mbuf. 2122 */ 2123 rxm->next = NULL; 2124 if (unlikely(rxq->crc_len > 0)) { 2125 first_seg->pkt_len -= RTE_ETHER_CRC_LEN; 2126 if (rx_packet_len <= RTE_ETHER_CRC_LEN) { 2127 rte_pktmbuf_free_seg(rxm); 2128 first_seg->nb_segs--; 2129 last_seg->data_len = 2130 (uint16_t)(last_seg->data_len - 2131 (RTE_ETHER_CRC_LEN - rx_packet_len)); 2132 last_seg->next = NULL; 2133 } else 2134 rxm->data_len = (uint16_t)(rx_packet_len - 2135 RTE_ETHER_CRC_LEN); 2136 } else if (rx_packet_len == 0) { 2137 rte_pktmbuf_free_seg(rxm); 2138 first_seg->nb_segs--; 2139 last_seg->next = NULL; 2140 } 2141 2142 first_seg->port = rxq->port_id; 2143 first_seg->ol_flags = 0; 2144 first_seg->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M & 2145 rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)]; 2146 ice_rxd_to_vlan_tci(first_seg, &rxd); 2147 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, first_seg, &rxd); 2148 pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0); 2149 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 2150 if (ice_timestamp_dynflag > 0 && 2151 (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { 2152 rxq->time_high = 2153 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high); 2154 if (unlikely(is_tsinit)) { 2155 ts_ns = ice_tstamp_convert_32b_64b(hw, ad, 1, rxq->time_high); 2156 rxq->hw_time_low = (uint32_t)ts_ns; 2157 rxq->hw_time_high = (uint32_t)(ts_ns >> 32); 2158 is_tsinit = false; 2159 } else { 2160 if (rxq->time_high < rxq->hw_time_low) 2161 rxq->hw_time_high += 1; 2162 ts_ns = (uint64_t)rxq->hw_time_high << 32 | rxq->time_high; 2163 rxq->hw_time_low = rxq->time_high; 2164 } 2165 rxq->hw_time_update = rte_get_timer_cycles() / 2166 (rte_get_timer_hz() / 1000); 2167 *RTE_MBUF_DYNFIELD(first_seg, 2168 (ice_timestamp_dynfield_offset), 2169 rte_mbuf_timestamp_t *) = ts_ns; 2170 pkt_flags |= ice_timestamp_dynflag; 2171 } 2172 2173 if (ad->ptp_ena && ((first_seg->packet_type & RTE_PTYPE_L2_MASK) 2174 == RTE_PTYPE_L2_ETHER_TIMESYNC)) { 2175 rxq->time_high = 2176 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high); 2177 first_seg->timesync = rxq->queue_id; 2178 pkt_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; 2179 } 2180 #endif 2181 first_seg->ol_flags |= pkt_flags; 2182 /* Prefetch data of first segment, if configured to do so. */ 2183 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr, 2184 first_seg->data_off)); 2185 rx_pkts[nb_rx++] = first_seg; 2186 first_seg = NULL; 2187 } 2188 2189 /* Record index of the next RX descriptor to probe. */ 2190 rxq->rx_tail = rx_id; 2191 rxq->pkt_first_seg = first_seg; 2192 rxq->pkt_last_seg = last_seg; 2193 2194 /** 2195 * If the number of free RX descriptors is greater than the RX free 2196 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 2197 * register. Update the RDT with the value of the last processed RX 2198 * descriptor minus 1, to guarantee that the RDT register is never 2199 * equal to the RDH register, which creates a "full" ring situation 2200 * from the hardware point of view. 2201 */ 2202 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 2203 if (nb_hold > rxq->rx_free_thresh) { 2204 rx_id = (uint16_t)(rx_id == 0 ? 2205 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 2206 /* write TAIL register */ 2207 ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 2208 nb_hold = 0; 2209 } 2210 rxq->nb_rx_hold = nb_hold; 2211 2212 /* return received packet in the burst */ 2213 return nb_rx; 2214 } 2215 2216 const uint32_t * 2217 ice_dev_supported_ptypes_get(struct rte_eth_dev *dev, size_t *no_of_elements) 2218 { 2219 struct ice_adapter *ad = 2220 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2221 const uint32_t *ptypes; 2222 2223 static const uint32_t ptypes_os[] = { 2224 /* refers to ice_get_default_pkt_type() */ 2225 RTE_PTYPE_L2_ETHER, 2226 RTE_PTYPE_L2_ETHER_TIMESYNC, 2227 RTE_PTYPE_L2_ETHER_LLDP, 2228 RTE_PTYPE_L2_ETHER_ARP, 2229 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 2230 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 2231 RTE_PTYPE_L4_FRAG, 2232 RTE_PTYPE_L4_ICMP, 2233 RTE_PTYPE_L4_NONFRAG, 2234 RTE_PTYPE_L4_SCTP, 2235 RTE_PTYPE_L4_TCP, 2236 RTE_PTYPE_L4_UDP, 2237 RTE_PTYPE_TUNNEL_GRENAT, 2238 RTE_PTYPE_TUNNEL_IP, 2239 RTE_PTYPE_INNER_L2_ETHER, 2240 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 2241 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 2242 RTE_PTYPE_INNER_L4_FRAG, 2243 RTE_PTYPE_INNER_L4_ICMP, 2244 RTE_PTYPE_INNER_L4_NONFRAG, 2245 RTE_PTYPE_INNER_L4_SCTP, 2246 RTE_PTYPE_INNER_L4_TCP, 2247 RTE_PTYPE_INNER_L4_UDP, 2248 }; 2249 2250 static const uint32_t ptypes_comms[] = { 2251 /* refers to ice_get_default_pkt_type() */ 2252 RTE_PTYPE_L2_ETHER, 2253 RTE_PTYPE_L2_ETHER_TIMESYNC, 2254 RTE_PTYPE_L2_ETHER_LLDP, 2255 RTE_PTYPE_L2_ETHER_ARP, 2256 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 2257 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, 2258 RTE_PTYPE_L4_FRAG, 2259 RTE_PTYPE_L4_ICMP, 2260 RTE_PTYPE_L4_NONFRAG, 2261 RTE_PTYPE_L4_SCTP, 2262 RTE_PTYPE_L4_TCP, 2263 RTE_PTYPE_L4_UDP, 2264 RTE_PTYPE_TUNNEL_GRENAT, 2265 RTE_PTYPE_TUNNEL_IP, 2266 RTE_PTYPE_INNER_L2_ETHER, 2267 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, 2268 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, 2269 RTE_PTYPE_INNER_L4_FRAG, 2270 RTE_PTYPE_INNER_L4_ICMP, 2271 RTE_PTYPE_INNER_L4_NONFRAG, 2272 RTE_PTYPE_INNER_L4_SCTP, 2273 RTE_PTYPE_INNER_L4_TCP, 2274 RTE_PTYPE_INNER_L4_UDP, 2275 RTE_PTYPE_TUNNEL_GTPC, 2276 RTE_PTYPE_TUNNEL_GTPU, 2277 RTE_PTYPE_L2_ETHER_PPPOE, 2278 }; 2279 2280 if (ad->active_pkg_type == ICE_PKG_TYPE_COMMS) { 2281 *no_of_elements = RTE_DIM(ptypes_comms); 2282 ptypes = ptypes_comms; 2283 } else { 2284 *no_of_elements = RTE_DIM(ptypes_os); 2285 ptypes = ptypes_os; 2286 } 2287 2288 if (dev->rx_pkt_burst == ice_recv_pkts || 2289 dev->rx_pkt_burst == ice_recv_pkts_bulk_alloc || 2290 dev->rx_pkt_burst == ice_recv_scattered_pkts) 2291 return ptypes; 2292 2293 #ifdef RTE_ARCH_X86 2294 if (dev->rx_pkt_burst == ice_recv_pkts_vec || 2295 dev->rx_pkt_burst == ice_recv_scattered_pkts_vec || 2296 #ifdef CC_AVX512_SUPPORT 2297 dev->rx_pkt_burst == ice_recv_pkts_vec_avx512 || 2298 dev->rx_pkt_burst == ice_recv_pkts_vec_avx512_offload || 2299 dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512 || 2300 dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx512_offload || 2301 #endif 2302 dev->rx_pkt_burst == ice_recv_pkts_vec_avx2 || 2303 dev->rx_pkt_burst == ice_recv_pkts_vec_avx2_offload || 2304 dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2 || 2305 dev->rx_pkt_burst == ice_recv_scattered_pkts_vec_avx2_offload) 2306 return ptypes; 2307 #endif 2308 2309 return NULL; 2310 } 2311 2312 int 2313 ice_rx_descriptor_status(void *rx_queue, uint16_t offset) 2314 { 2315 volatile union ice_rx_flex_desc *rxdp; 2316 struct ice_rx_queue *rxq = rx_queue; 2317 uint32_t desc; 2318 2319 if (unlikely(offset >= rxq->nb_rx_desc)) 2320 return -EINVAL; 2321 2322 if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) 2323 return RTE_ETH_RX_DESC_UNAVAIL; 2324 2325 desc = rxq->rx_tail + offset; 2326 if (desc >= rxq->nb_rx_desc) 2327 desc -= rxq->nb_rx_desc; 2328 2329 rxdp = &rxq->rx_ring[desc]; 2330 if (rte_le_to_cpu_16(rxdp->wb.status_error0) & 2331 (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)) 2332 return RTE_ETH_RX_DESC_DONE; 2333 2334 return RTE_ETH_RX_DESC_AVAIL; 2335 } 2336 2337 int 2338 ice_tx_descriptor_status(void *tx_queue, uint16_t offset) 2339 { 2340 struct ci_tx_queue *txq = tx_queue; 2341 volatile uint64_t *status; 2342 uint64_t mask, expect; 2343 uint32_t desc; 2344 2345 if (unlikely(offset >= txq->nb_tx_desc)) 2346 return -EINVAL; 2347 2348 desc = txq->tx_tail + offset; 2349 /* go to next desc that has the RS bit */ 2350 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * 2351 txq->tx_rs_thresh; 2352 if (desc >= txq->nb_tx_desc) { 2353 desc -= txq->nb_tx_desc; 2354 if (desc >= txq->nb_tx_desc) 2355 desc -= txq->nb_tx_desc; 2356 } 2357 2358 status = &txq->ice_tx_ring[desc].cmd_type_offset_bsz; 2359 mask = rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M); 2360 expect = rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE << 2361 ICE_TXD_QW1_DTYPE_S); 2362 if ((*status & mask) == expect) 2363 return RTE_ETH_TX_DESC_DONE; 2364 2365 return RTE_ETH_TX_DESC_FULL; 2366 } 2367 2368 void 2369 ice_free_queues(struct rte_eth_dev *dev) 2370 { 2371 uint16_t i; 2372 2373 PMD_INIT_FUNC_TRACE(); 2374 2375 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2376 if (!dev->data->rx_queues[i]) 2377 continue; 2378 ice_rx_queue_release(dev->data->rx_queues[i]); 2379 dev->data->rx_queues[i] = NULL; 2380 } 2381 dev->data->nb_rx_queues = 0; 2382 2383 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2384 if (!dev->data->tx_queues[i]) 2385 continue; 2386 ice_tx_queue_release(dev->data->tx_queues[i]); 2387 dev->data->tx_queues[i] = NULL; 2388 } 2389 dev->data->nb_tx_queues = 0; 2390 } 2391 2392 #define ICE_FDIR_NUM_TX_DESC ICE_MIN_RING_DESC 2393 #define ICE_FDIR_NUM_RX_DESC ICE_MIN_RING_DESC 2394 2395 int 2396 ice_fdir_setup_tx_resources(struct ice_pf *pf) 2397 { 2398 struct ci_tx_queue *txq; 2399 const struct rte_memzone *tz = NULL; 2400 uint32_t ring_size; 2401 struct rte_eth_dev *dev; 2402 2403 if (!pf) { 2404 PMD_DRV_LOG(ERR, "PF is not available"); 2405 return -EINVAL; 2406 } 2407 2408 dev = &rte_eth_devices[pf->adapter->pf.dev_data->port_id]; 2409 2410 /* Allocate the TX queue data structure. */ 2411 txq = rte_zmalloc_socket("ice fdir tx queue", 2412 sizeof(struct ci_tx_queue), 2413 RTE_CACHE_LINE_SIZE, 2414 SOCKET_ID_ANY); 2415 if (!txq) { 2416 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 2417 "tx queue structure."); 2418 return -ENOMEM; 2419 } 2420 2421 /* Allocate TX hardware ring descriptors. */ 2422 ring_size = sizeof(struct ice_tx_desc) * ICE_FDIR_NUM_TX_DESC; 2423 ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN); 2424 2425 tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring", 2426 ICE_FDIR_QUEUE_ID, ring_size, 2427 ICE_RING_BASE_ALIGN, SOCKET_ID_ANY); 2428 if (!tz) { 2429 ice_tx_queue_release(txq); 2430 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX."); 2431 return -ENOMEM; 2432 } 2433 2434 txq->mz = tz; 2435 txq->nb_tx_desc = ICE_FDIR_NUM_TX_DESC; 2436 txq->queue_id = ICE_FDIR_QUEUE_ID; 2437 txq->reg_idx = pf->fdir.fdir_vsi->base_queue; 2438 txq->ice_vsi = pf->fdir.fdir_vsi; 2439 2440 txq->tx_ring_dma = tz->iova; 2441 txq->ice_tx_ring = (struct ice_tx_desc *)tz->addr; 2442 /* 2443 * don't need to allocate software ring and reset for the fdir 2444 * program queue just set the queue has been configured. 2445 */ 2446 txq->q_set = true; 2447 pf->fdir.txq = txq; 2448 2449 2450 return ICE_SUCCESS; 2451 } 2452 2453 int 2454 ice_fdir_setup_rx_resources(struct ice_pf *pf) 2455 { 2456 struct ice_rx_queue *rxq; 2457 const struct rte_memzone *rz = NULL; 2458 uint32_t ring_size; 2459 struct rte_eth_dev *dev; 2460 2461 if (!pf) { 2462 PMD_DRV_LOG(ERR, "PF is not available"); 2463 return -EINVAL; 2464 } 2465 2466 dev = &rte_eth_devices[pf->adapter->pf.dev_data->port_id]; 2467 2468 /* Allocate the RX queue data structure. */ 2469 rxq = rte_zmalloc_socket("ice fdir rx queue", 2470 sizeof(struct ice_rx_queue), 2471 RTE_CACHE_LINE_SIZE, 2472 SOCKET_ID_ANY); 2473 if (!rxq) { 2474 PMD_DRV_LOG(ERR, "Failed to allocate memory for " 2475 "rx queue structure."); 2476 return -ENOMEM; 2477 } 2478 2479 /* Allocate RX hardware ring descriptors. */ 2480 ring_size = sizeof(union ice_32byte_rx_desc) * ICE_FDIR_NUM_RX_DESC; 2481 ring_size = RTE_ALIGN(ring_size, ICE_DMA_MEM_ALIGN); 2482 2483 rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring", 2484 ICE_FDIR_QUEUE_ID, ring_size, 2485 ICE_RING_BASE_ALIGN, SOCKET_ID_ANY); 2486 if (!rz) { 2487 ice_rx_queue_release(rxq); 2488 PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX."); 2489 return -ENOMEM; 2490 } 2491 2492 rxq->mz = rz; 2493 rxq->nb_rx_desc = ICE_FDIR_NUM_RX_DESC; 2494 rxq->queue_id = ICE_FDIR_QUEUE_ID; 2495 rxq->reg_idx = pf->fdir.fdir_vsi->base_queue; 2496 rxq->vsi = pf->fdir.fdir_vsi; 2497 2498 rxq->rx_ring_dma = rz->iova; 2499 memset(rz->addr, 0, ICE_FDIR_NUM_RX_DESC * 2500 sizeof(union ice_32byte_rx_desc)); 2501 rxq->rx_ring = (union ice_rx_flex_desc *)rz->addr; 2502 2503 /* 2504 * Don't need to allocate software ring and reset for the fdir 2505 * rx queue, just set the queue has been configured. 2506 */ 2507 rxq->q_set = true; 2508 pf->fdir.rxq = rxq; 2509 2510 rxq->rx_rel_mbufs = _ice_rx_queue_release_mbufs; 2511 2512 return ICE_SUCCESS; 2513 } 2514 2515 uint16_t 2516 ice_recv_pkts(void *rx_queue, 2517 struct rte_mbuf **rx_pkts, 2518 uint16_t nb_pkts) 2519 { 2520 struct ice_rx_queue *rxq = rx_queue; 2521 volatile union ice_rx_flex_desc *rx_ring = rxq->rx_ring; 2522 volatile union ice_rx_flex_desc *rxdp; 2523 union ice_rx_flex_desc rxd; 2524 struct ice_rx_entry *sw_ring = rxq->sw_ring; 2525 struct ice_rx_entry *rxe; 2526 struct rte_mbuf *nmb; /* new allocated mbuf */ 2527 struct rte_mbuf *nmb_pay; /* new allocated payload mbuf */ 2528 struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */ 2529 uint16_t rx_id = rxq->rx_tail; 2530 uint16_t nb_rx = 0; 2531 uint16_t nb_hold = 0; 2532 uint16_t rx_packet_len; 2533 uint16_t rx_header_len; 2534 uint16_t rx_stat_err0; 2535 uint64_t dma_addr; 2536 uint64_t pkt_flags; 2537 uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl; 2538 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 2539 bool is_tsinit = false; 2540 uint64_t ts_ns; 2541 struct ice_vsi *vsi = rxq->vsi; 2542 struct ice_hw *hw = ICE_VSI_TO_HW(vsi); 2543 struct ice_adapter *ad = rxq->vsi->adapter; 2544 2545 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) { 2546 uint64_t sw_cur_time = rte_get_timer_cycles() / (rte_get_timer_hz() / 1000); 2547 2548 if (unlikely(sw_cur_time - rxq->hw_time_update > 4)) 2549 is_tsinit = 1; 2550 } 2551 #endif 2552 2553 while (nb_rx < nb_pkts) { 2554 rxdp = &rx_ring[rx_id]; 2555 rx_stat_err0 = rte_le_to_cpu_16(rxdp->wb.status_error0); 2556 2557 /* Check the DD bit first */ 2558 if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S))) 2559 break; 2560 2561 /* allocate header mbuf */ 2562 nmb = rte_mbuf_raw_alloc(rxq->mp); 2563 if (unlikely(!nmb)) { 2564 rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++; 2565 break; 2566 } 2567 2568 rxd = *rxdp; /* copy descriptor in ring to temp variable*/ 2569 2570 nb_hold++; 2571 rxe = &sw_ring[rx_id]; /* get corresponding mbuf in SW ring */ 2572 rx_id++; 2573 if (unlikely(rx_id == rxq->nb_rx_desc)) 2574 rx_id = 0; 2575 rxm = rxe->mbuf; 2576 rxe->mbuf = nmb; 2577 dma_addr = 2578 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 2579 2580 if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2581 /** 2582 * fill the read format of descriptor with physic address in 2583 * new allocated mbuf: nmb 2584 */ 2585 rxdp->read.hdr_addr = 0; 2586 rxdp->read.pkt_addr = dma_addr; 2587 } else { 2588 /* allocate payload mbuf */ 2589 nmb_pay = rte_mbuf_raw_alloc(rxq->rxseg[1].mp); 2590 if (unlikely(!nmb_pay)) { 2591 rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++; 2592 rxe->mbuf = NULL; 2593 nb_hold--; 2594 if (unlikely(rx_id == 0)) 2595 rx_id = rxq->nb_rx_desc; 2596 2597 rx_id--; 2598 rte_pktmbuf_free(nmb); 2599 break; 2600 } 2601 2602 nmb->next = nmb_pay; 2603 nmb_pay->next = NULL; 2604 2605 /** 2606 * fill the read format of descriptor with physic address in 2607 * new allocated mbuf: nmb 2608 */ 2609 rxdp->read.hdr_addr = dma_addr; 2610 rxdp->read.pkt_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb_pay)); 2611 } 2612 2613 /* fill old mbuf with received descriptor: rxd */ 2614 rxm->data_off = RTE_PKTMBUF_HEADROOM; 2615 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM)); 2616 if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 2617 rxm->nb_segs = 1; 2618 rxm->next = NULL; 2619 /* calculate rx_packet_len of the received pkt */ 2620 rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) & 2621 ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 2622 rxm->data_len = rx_packet_len; 2623 rxm->pkt_len = rx_packet_len; 2624 } else { 2625 rxm->nb_segs = (uint16_t)(rxm->nb_segs + rxm->next->nb_segs); 2626 rxm->next->next = NULL; 2627 /* calculate rx_packet_len of the received pkt */ 2628 rx_header_len = rte_le_to_cpu_16(rxd.wb.hdr_len_sph_flex_flags1) & 2629 ICE_RX_FLEX_DESC_HEADER_LEN_M; 2630 rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) & 2631 ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len; 2632 rxm->data_len = rx_header_len; 2633 rxm->pkt_len = rx_header_len + rx_packet_len; 2634 rxm->next->data_len = rx_packet_len; 2635 2636 #ifdef RTE_ETHDEV_DEBUG_RX 2637 rte_pktmbuf_dump(stdout, rxm, rte_pktmbuf_pkt_len(rxm)); 2638 #endif 2639 } 2640 2641 rxm->port = rxq->port_id; 2642 rxm->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M & 2643 rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)]; 2644 ice_rxd_to_vlan_tci(rxm, &rxd); 2645 rxd_to_pkt_fields_ops[rxq->rxdid](rxq, rxm, &rxd); 2646 pkt_flags = ice_rxd_error_to_pkt_flags(rx_stat_err0); 2647 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 2648 if (ice_timestamp_dynflag > 0 && 2649 (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { 2650 rxq->time_high = 2651 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high); 2652 if (unlikely(is_tsinit)) { 2653 ts_ns = ice_tstamp_convert_32b_64b(hw, ad, 1, rxq->time_high); 2654 rxq->hw_time_low = (uint32_t)ts_ns; 2655 rxq->hw_time_high = (uint32_t)(ts_ns >> 32); 2656 is_tsinit = false; 2657 } else { 2658 if (rxq->time_high < rxq->hw_time_low) 2659 rxq->hw_time_high += 1; 2660 ts_ns = (uint64_t)rxq->hw_time_high << 32 | rxq->time_high; 2661 rxq->hw_time_low = rxq->time_high; 2662 } 2663 rxq->hw_time_update = rte_get_timer_cycles() / 2664 (rte_get_timer_hz() / 1000); 2665 *RTE_MBUF_DYNFIELD(rxm, 2666 (ice_timestamp_dynfield_offset), 2667 rte_mbuf_timestamp_t *) = ts_ns; 2668 pkt_flags |= ice_timestamp_dynflag; 2669 } 2670 2671 if (ad->ptp_ena && ((rxm->packet_type & RTE_PTYPE_L2_MASK) == 2672 RTE_PTYPE_L2_ETHER_TIMESYNC)) { 2673 rxq->time_high = 2674 rte_le_to_cpu_32(rxd.wb.flex_ts.ts_high); 2675 rxm->timesync = rxq->queue_id; 2676 pkt_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; 2677 } 2678 #endif 2679 rxm->ol_flags |= pkt_flags; 2680 /* copy old mbuf to rx_pkts */ 2681 rx_pkts[nb_rx++] = rxm; 2682 } 2683 2684 rxq->rx_tail = rx_id; 2685 /** 2686 * If the number of free RX descriptors is greater than the RX free 2687 * threshold of the queue, advance the receive tail register of queue. 2688 * Update that register with the value of the last processed RX 2689 * descriptor minus 1. 2690 */ 2691 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 2692 if (nb_hold > rxq->rx_free_thresh) { 2693 rx_id = (uint16_t)(rx_id == 0 ? 2694 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 2695 /* write TAIL register */ 2696 ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id); 2697 nb_hold = 0; 2698 } 2699 rxq->nb_rx_hold = nb_hold; 2700 2701 /* return received packet in the burst */ 2702 return nb_rx; 2703 } 2704 2705 static inline void 2706 ice_parse_tunneling_params(uint64_t ol_flags, 2707 union ice_tx_offload tx_offload, 2708 uint32_t *cd_tunneling) 2709 { 2710 /* EIPT: External (outer) IP header type */ 2711 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) 2712 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV4; 2713 else if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) 2714 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV4_NO_CSUM; 2715 else if (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6) 2716 *cd_tunneling |= ICE_TX_CTX_EIPT_IPV6; 2717 2718 /* EIPLEN: External (outer) IP header length, in DWords */ 2719 *cd_tunneling |= (tx_offload.outer_l3_len >> 2) << 2720 ICE_TXD_CTX_QW0_EIPLEN_S; 2721 2722 /* L4TUNT: L4 Tunneling Type */ 2723 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 2724 case RTE_MBUF_F_TX_TUNNEL_IPIP: 2725 /* for non UDP / GRE tunneling, set to 00b */ 2726 break; 2727 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 2728 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: 2729 case RTE_MBUF_F_TX_TUNNEL_GTP: 2730 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 2731 *cd_tunneling |= ICE_TXD_CTX_UDP_TUNNELING; 2732 break; 2733 case RTE_MBUF_F_TX_TUNNEL_GRE: 2734 *cd_tunneling |= ICE_TXD_CTX_GRE_TUNNELING; 2735 break; 2736 default: 2737 PMD_TX_LOG(ERR, "Tunnel type not supported"); 2738 return; 2739 } 2740 2741 /* L4TUNLEN: L4 Tunneling Length, in Words 2742 * 2743 * We depend on app to set rte_mbuf.l2_len correctly. 2744 * For IP in GRE it should be set to the length of the GRE 2745 * header; 2746 * For MAC in GRE or MAC in UDP it should be set to the length 2747 * of the GRE or UDP headers plus the inner MAC up to including 2748 * its last Ethertype. 2749 * If MPLS labels exists, it should include them as well. 2750 */ 2751 *cd_tunneling |= (tx_offload.l2_len >> 1) << 2752 ICE_TXD_CTX_QW0_NATLEN_S; 2753 2754 /** 2755 * Calculate the tunneling UDP checksum. 2756 * Shall be set only if L4TUNT = 01b and EIPT is not zero 2757 */ 2758 if ((*cd_tunneling & ICE_TXD_CTX_QW0_EIPT_M) && 2759 (*cd_tunneling & ICE_TXD_CTX_UDP_TUNNELING) && 2760 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) 2761 *cd_tunneling |= ICE_TXD_CTX_QW0_L4T_CS_M; 2762 } 2763 2764 static inline void 2765 ice_txd_enable_checksum(uint64_t ol_flags, 2766 uint32_t *td_cmd, 2767 uint32_t *td_offset, 2768 union ice_tx_offload tx_offload) 2769 { 2770 /* Set MACLEN */ 2771 if (!(ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK)) 2772 *td_offset |= (tx_offload.l2_len >> 1) 2773 << ICE_TX_DESC_LEN_MACLEN_S; 2774 2775 /* Enable L3 checksum offloads */ 2776 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 2777 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 2778 *td_offset |= (tx_offload.l3_len >> 2) << 2779 ICE_TX_DESC_LEN_IPLEN_S; 2780 } else if (ol_flags & RTE_MBUF_F_TX_IPV4) { 2781 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 2782 *td_offset |= (tx_offload.l3_len >> 2) << 2783 ICE_TX_DESC_LEN_IPLEN_S; 2784 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) { 2785 *td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 2786 *td_offset |= (tx_offload.l3_len >> 2) << 2787 ICE_TX_DESC_LEN_IPLEN_S; 2788 } 2789 2790 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 2791 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 2792 *td_offset |= (tx_offload.l4_len >> 2) << 2793 ICE_TX_DESC_LEN_L4_LEN_S; 2794 return; 2795 } 2796 2797 if (ol_flags & RTE_MBUF_F_TX_UDP_SEG) { 2798 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 2799 *td_offset |= (tx_offload.l4_len >> 2) << 2800 ICE_TX_DESC_LEN_L4_LEN_S; 2801 return; 2802 } 2803 2804 /* Enable L4 checksum offloads */ 2805 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 2806 case RTE_MBUF_F_TX_TCP_CKSUM: 2807 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 2808 *td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) << 2809 ICE_TX_DESC_LEN_L4_LEN_S; 2810 break; 2811 case RTE_MBUF_F_TX_SCTP_CKSUM: 2812 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 2813 *td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) << 2814 ICE_TX_DESC_LEN_L4_LEN_S; 2815 break; 2816 case RTE_MBUF_F_TX_UDP_CKSUM: 2817 *td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 2818 *td_offset |= (sizeof(struct rte_udp_hdr) >> 2) << 2819 ICE_TX_DESC_LEN_L4_LEN_S; 2820 break; 2821 default: 2822 break; 2823 } 2824 } 2825 2826 static inline int 2827 ice_xmit_cleanup(struct ci_tx_queue *txq) 2828 { 2829 struct ci_tx_entry *sw_ring = txq->sw_ring; 2830 volatile struct ice_tx_desc *txd = txq->ice_tx_ring; 2831 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 2832 uint16_t nb_tx_desc = txq->nb_tx_desc; 2833 uint16_t desc_to_clean_to; 2834 uint16_t nb_tx_to_clean; 2835 2836 /* Determine the last descriptor needing to be cleaned */ 2837 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh); 2838 if (desc_to_clean_to >= nb_tx_desc) 2839 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 2840 2841 /* Check to make sure the last descriptor to clean is done */ 2842 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 2843 if (!(txd[desc_to_clean_to].cmd_type_offset_bsz & 2844 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))) { 2845 PMD_TX_LOG(DEBUG, "TX descriptor %4u is not done " 2846 "(port=%d queue=%d) value=0x%"PRIx64, 2847 desc_to_clean_to, 2848 txq->port_id, txq->queue_id, 2849 txd[desc_to_clean_to].cmd_type_offset_bsz); 2850 /* Failed to clean any descriptors */ 2851 return -1; 2852 } 2853 2854 /* Figure out how many descriptors will be cleaned */ 2855 if (last_desc_cleaned > desc_to_clean_to) 2856 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 2857 desc_to_clean_to); 2858 else 2859 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 2860 last_desc_cleaned); 2861 2862 /* The last descriptor to clean is done, so that means all the 2863 * descriptors from the last descriptor that was cleaned 2864 * up to the last descriptor with the RS bit set 2865 * are done. Only reset the threshold descriptor. 2866 */ 2867 txd[desc_to_clean_to].cmd_type_offset_bsz = 0; 2868 2869 /* Update the txq to reflect the last descriptor that was cleaned */ 2870 txq->last_desc_cleaned = desc_to_clean_to; 2871 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 2872 2873 return 0; 2874 } 2875 2876 /* Construct the tx flags */ 2877 static inline uint64_t 2878 ice_build_ctob(uint32_t td_cmd, 2879 uint32_t td_offset, 2880 uint16_t size, 2881 uint32_t td_tag) 2882 { 2883 return rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA | 2884 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) | 2885 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) | 2886 ((uint64_t)size << ICE_TXD_QW1_TX_BUF_SZ_S) | 2887 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S)); 2888 } 2889 2890 /* Check if the context descriptor is needed for TX offloading */ 2891 static inline uint16_t 2892 ice_calc_context_desc(uint64_t flags) 2893 { 2894 static uint64_t mask = RTE_MBUF_F_TX_TCP_SEG | 2895 RTE_MBUF_F_TX_UDP_SEG | 2896 RTE_MBUF_F_TX_QINQ | 2897 RTE_MBUF_F_TX_OUTER_IP_CKSUM | 2898 RTE_MBUF_F_TX_TUNNEL_MASK | 2899 RTE_MBUF_F_TX_IEEE1588_TMST; 2900 2901 return (flags & mask) ? 1 : 0; 2902 } 2903 2904 /* set ice TSO context descriptor */ 2905 static inline uint64_t 2906 ice_set_tso_ctx(struct rte_mbuf *mbuf, union ice_tx_offload tx_offload) 2907 { 2908 uint64_t ctx_desc = 0; 2909 uint32_t cd_cmd, hdr_len, cd_tso_len; 2910 2911 if (!tx_offload.l4_len) { 2912 PMD_TX_LOG(DEBUG, "L4 length set to 0"); 2913 return ctx_desc; 2914 } 2915 2916 hdr_len = tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len; 2917 hdr_len += (mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? 2918 tx_offload.outer_l2_len + tx_offload.outer_l3_len : 0; 2919 2920 cd_cmd = ICE_TX_CTX_DESC_TSO; 2921 cd_tso_len = mbuf->pkt_len - hdr_len; 2922 ctx_desc |= ((uint64_t)cd_cmd << ICE_TXD_CTX_QW1_CMD_S) | 2923 ((uint64_t)cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) | 2924 ((uint64_t)mbuf->tso_segsz << ICE_TXD_CTX_QW1_MSS_S); 2925 2926 return ctx_desc; 2927 } 2928 2929 /* HW requires that TX buffer size ranges from 1B up to (16K-1)B. */ 2930 #define ICE_MAX_DATA_PER_TXD \ 2931 (ICE_TXD_QW1_TX_BUF_SZ_M >> ICE_TXD_QW1_TX_BUF_SZ_S) 2932 /* Calculate the number of TX descriptors needed for each pkt */ 2933 static inline uint16_t 2934 ice_calc_pkt_desc(struct rte_mbuf *tx_pkt) 2935 { 2936 struct rte_mbuf *txd = tx_pkt; 2937 uint16_t count = 0; 2938 2939 while (txd != NULL) { 2940 count += DIV_ROUND_UP(txd->data_len, ICE_MAX_DATA_PER_TXD); 2941 txd = txd->next; 2942 } 2943 2944 return count; 2945 } 2946 2947 uint16_t 2948 ice_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2949 { 2950 struct ci_tx_queue *txq; 2951 volatile struct ice_tx_desc *ice_tx_ring; 2952 volatile struct ice_tx_desc *txd; 2953 struct ci_tx_entry *sw_ring; 2954 struct ci_tx_entry *txe, *txn; 2955 struct rte_mbuf *tx_pkt; 2956 struct rte_mbuf *m_seg; 2957 uint32_t cd_tunneling_params; 2958 uint16_t tx_id; 2959 uint16_t nb_tx; 2960 uint16_t nb_used; 2961 uint16_t nb_ctx; 2962 uint32_t td_cmd = 0; 2963 uint32_t td_offset = 0; 2964 uint32_t td_tag = 0; 2965 uint16_t tx_last; 2966 uint16_t slen; 2967 uint64_t buf_dma_addr; 2968 uint64_t ol_flags; 2969 union ice_tx_offload tx_offload = {0}; 2970 2971 txq = tx_queue; 2972 sw_ring = txq->sw_ring; 2973 ice_tx_ring = txq->ice_tx_ring; 2974 tx_id = txq->tx_tail; 2975 txe = &sw_ring[tx_id]; 2976 2977 /* Check if the descriptor ring needs to be cleaned. */ 2978 if (txq->nb_tx_free < txq->tx_free_thresh) 2979 (void)ice_xmit_cleanup(txq); 2980 2981 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 2982 tx_pkt = *tx_pkts++; 2983 2984 td_cmd = 0; 2985 td_tag = 0; 2986 td_offset = 0; 2987 ol_flags = tx_pkt->ol_flags; 2988 tx_offload.l2_len = tx_pkt->l2_len; 2989 tx_offload.l3_len = tx_pkt->l3_len; 2990 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 2991 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 2992 tx_offload.l4_len = tx_pkt->l4_len; 2993 tx_offload.tso_segsz = tx_pkt->tso_segsz; 2994 /* Calculate the number of context descriptors needed. */ 2995 nb_ctx = ice_calc_context_desc(ol_flags); 2996 2997 /* The number of descriptors that must be allocated for 2998 * a packet equals to the number of the segments of that 2999 * packet plus the number of context descriptor if needed. 3000 * Recalculate the needed tx descs when TSO enabled in case 3001 * the mbuf data size exceeds max data size that hw allows 3002 * per tx desc. 3003 */ 3004 if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) 3005 nb_used = (uint16_t)(ice_calc_pkt_desc(tx_pkt) + 3006 nb_ctx); 3007 else 3008 nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx); 3009 tx_last = (uint16_t)(tx_id + nb_used - 1); 3010 3011 /* Circular ring */ 3012 if (tx_last >= txq->nb_tx_desc) 3013 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 3014 3015 if (nb_used > txq->nb_tx_free) { 3016 if (ice_xmit_cleanup(txq) != 0) { 3017 if (nb_tx == 0) 3018 return 0; 3019 goto end_of_tx; 3020 } 3021 if (unlikely(nb_used > txq->tx_rs_thresh)) { 3022 while (nb_used > txq->nb_tx_free) { 3023 if (ice_xmit_cleanup(txq) != 0) { 3024 if (nb_tx == 0) 3025 return 0; 3026 goto end_of_tx; 3027 } 3028 } 3029 } 3030 } 3031 3032 /* Descriptor based VLAN insertion */ 3033 if (ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { 3034 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1; 3035 td_tag = tx_pkt->vlan_tci; 3036 } 3037 3038 /* Fill in tunneling parameters if necessary */ 3039 cd_tunneling_params = 0; 3040 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 3041 td_offset |= (tx_offload.outer_l2_len >> 1) 3042 << ICE_TX_DESC_LEN_MACLEN_S; 3043 ice_parse_tunneling_params(ol_flags, tx_offload, 3044 &cd_tunneling_params); 3045 } 3046 3047 /* Enable checksum offloading */ 3048 if (ol_flags & ICE_TX_CKSUM_OFFLOAD_MASK) 3049 ice_txd_enable_checksum(ol_flags, &td_cmd, 3050 &td_offset, tx_offload); 3051 3052 if (nb_ctx) { 3053 /* Setup TX context descriptor if required */ 3054 volatile struct ice_tx_ctx_desc *ctx_txd = 3055 (volatile struct ice_tx_ctx_desc *) 3056 &ice_tx_ring[tx_id]; 3057 uint16_t cd_l2tag2 = 0; 3058 uint64_t cd_type_cmd_tso_mss = ICE_TX_DESC_DTYPE_CTX; 3059 3060 txn = &sw_ring[txe->next_id]; 3061 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 3062 if (txe->mbuf) { 3063 rte_pktmbuf_free_seg(txe->mbuf); 3064 txe->mbuf = NULL; 3065 } 3066 3067 if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) 3068 cd_type_cmd_tso_mss |= 3069 ice_set_tso_ctx(tx_pkt, tx_offload); 3070 else if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 3071 cd_type_cmd_tso_mss |= 3072 ((uint64_t)ICE_TX_CTX_DESC_TSYN << 3073 ICE_TXD_CTX_QW1_CMD_S) | 3074 (((uint64_t)txq->ice_vsi->adapter->ptp_tx_index << 3075 ICE_TXD_CTX_QW1_TSYN_S) & ICE_TXD_CTX_QW1_TSYN_M); 3076 3077 ctx_txd->tunneling_params = 3078 rte_cpu_to_le_32(cd_tunneling_params); 3079 3080 /* TX context descriptor based double VLAN insert */ 3081 if (ol_flags & RTE_MBUF_F_TX_QINQ) { 3082 cd_l2tag2 = tx_pkt->vlan_tci_outer; 3083 cd_type_cmd_tso_mss |= 3084 ((uint64_t)ICE_TX_CTX_DESC_IL2TAG2 << 3085 ICE_TXD_CTX_QW1_CMD_S); 3086 } 3087 ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2); 3088 ctx_txd->qw1 = 3089 rte_cpu_to_le_64(cd_type_cmd_tso_mss); 3090 3091 txe->last_id = tx_last; 3092 tx_id = txe->next_id; 3093 txe = txn; 3094 } 3095 m_seg = tx_pkt; 3096 3097 do { 3098 txd = &ice_tx_ring[tx_id]; 3099 txn = &sw_ring[txe->next_id]; 3100 3101 if (txe->mbuf) 3102 rte_pktmbuf_free_seg(txe->mbuf); 3103 txe->mbuf = m_seg; 3104 3105 /* Setup TX Descriptor */ 3106 slen = m_seg->data_len; 3107 buf_dma_addr = rte_mbuf_data_iova(m_seg); 3108 3109 while ((ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) && 3110 unlikely(slen > ICE_MAX_DATA_PER_TXD)) { 3111 txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr); 3112 txd->cmd_type_offset_bsz = 3113 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA | 3114 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) | 3115 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) | 3116 ((uint64_t)ICE_MAX_DATA_PER_TXD << 3117 ICE_TXD_QW1_TX_BUF_SZ_S) | 3118 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S)); 3119 3120 buf_dma_addr += ICE_MAX_DATA_PER_TXD; 3121 slen -= ICE_MAX_DATA_PER_TXD; 3122 3123 txe->last_id = tx_last; 3124 tx_id = txe->next_id; 3125 txe = txn; 3126 txd = &ice_tx_ring[tx_id]; 3127 txn = &sw_ring[txe->next_id]; 3128 } 3129 3130 txd->buf_addr = rte_cpu_to_le_64(buf_dma_addr); 3131 txd->cmd_type_offset_bsz = 3132 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DATA | 3133 ((uint64_t)td_cmd << ICE_TXD_QW1_CMD_S) | 3134 ((uint64_t)td_offset << ICE_TXD_QW1_OFFSET_S) | 3135 ((uint64_t)slen << ICE_TXD_QW1_TX_BUF_SZ_S) | 3136 ((uint64_t)td_tag << ICE_TXD_QW1_L2TAG1_S)); 3137 3138 txe->last_id = tx_last; 3139 tx_id = txe->next_id; 3140 txe = txn; 3141 m_seg = m_seg->next; 3142 } while (m_seg); 3143 3144 /* fill the last descriptor with End of Packet (EOP) bit */ 3145 td_cmd |= ICE_TX_DESC_CMD_EOP; 3146 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used); 3147 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 3148 3149 /* set RS bit on the last descriptor of one packet */ 3150 if (txq->nb_tx_used >= txq->tx_rs_thresh) { 3151 PMD_TX_LOG(DEBUG, 3152 "Setting RS bit on TXD id=" 3153 "%4u (port=%d queue=%d)", 3154 tx_last, txq->port_id, txq->queue_id); 3155 3156 td_cmd |= ICE_TX_DESC_CMD_RS; 3157 3158 /* Update txq RS bit counters */ 3159 txq->nb_tx_used = 0; 3160 } 3161 txd->cmd_type_offset_bsz |= 3162 rte_cpu_to_le_64(((uint64_t)td_cmd) << 3163 ICE_TXD_QW1_CMD_S); 3164 } 3165 end_of_tx: 3166 /* update Tail register */ 3167 ICE_PCI_REG_WRITE(txq->qtx_tail, tx_id); 3168 txq->tx_tail = tx_id; 3169 3170 return nb_tx; 3171 } 3172 3173 static __rte_always_inline int 3174 ice_tx_free_bufs(struct ci_tx_queue *txq) 3175 { 3176 struct ci_tx_entry *txep; 3177 uint16_t i; 3178 3179 if ((txq->ice_tx_ring[txq->tx_next_dd].cmd_type_offset_bsz & 3180 rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) != 3181 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) 3182 return 0; 3183 3184 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]; 3185 3186 for (i = 0; i < txq->tx_rs_thresh; i++) 3187 rte_prefetch0((txep + i)->mbuf); 3188 3189 if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { 3190 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { 3191 rte_mempool_put(txep->mbuf->pool, txep->mbuf); 3192 txep->mbuf = NULL; 3193 } 3194 } else { 3195 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { 3196 rte_pktmbuf_free_seg(txep->mbuf); 3197 txep->mbuf = NULL; 3198 } 3199 } 3200 3201 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 3202 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 3203 if (txq->tx_next_dd >= txq->nb_tx_desc) 3204 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 3205 3206 return txq->tx_rs_thresh; 3207 } 3208 3209 static int 3210 ice_tx_done_cleanup_full(struct ci_tx_queue *txq, 3211 uint32_t free_cnt) 3212 { 3213 struct ci_tx_entry *swr_ring = txq->sw_ring; 3214 uint16_t i, tx_last, tx_id; 3215 uint16_t nb_tx_free_last; 3216 uint16_t nb_tx_to_clean; 3217 uint32_t pkt_cnt; 3218 3219 /* Start free mbuf from the next of tx_tail */ 3220 tx_last = txq->tx_tail; 3221 tx_id = swr_ring[tx_last].next_id; 3222 3223 if (txq->nb_tx_free == 0 && ice_xmit_cleanup(txq)) 3224 return 0; 3225 3226 nb_tx_to_clean = txq->nb_tx_free; 3227 nb_tx_free_last = txq->nb_tx_free; 3228 if (!free_cnt) 3229 free_cnt = txq->nb_tx_desc; 3230 3231 /* Loop through swr_ring to count the amount of 3232 * freeable mubfs and packets. 3233 */ 3234 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 3235 for (i = 0; i < nb_tx_to_clean && 3236 pkt_cnt < free_cnt && 3237 tx_id != tx_last; i++) { 3238 if (swr_ring[tx_id].mbuf != NULL) { 3239 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 3240 swr_ring[tx_id].mbuf = NULL; 3241 3242 /* 3243 * last segment in the packet, 3244 * increment packet count 3245 */ 3246 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 3247 } 3248 3249 tx_id = swr_ring[tx_id].next_id; 3250 } 3251 3252 if (txq->tx_rs_thresh > txq->nb_tx_desc - 3253 txq->nb_tx_free || tx_id == tx_last) 3254 break; 3255 3256 if (pkt_cnt < free_cnt) { 3257 if (ice_xmit_cleanup(txq)) 3258 break; 3259 3260 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 3261 nb_tx_free_last = txq->nb_tx_free; 3262 } 3263 } 3264 3265 return (int)pkt_cnt; 3266 } 3267 3268 #ifdef RTE_ARCH_X86 3269 static int 3270 ice_tx_done_cleanup_vec(struct ci_tx_queue *txq __rte_unused, 3271 uint32_t free_cnt __rte_unused) 3272 { 3273 return -ENOTSUP; 3274 } 3275 #endif 3276 3277 static int 3278 ice_tx_done_cleanup_simple(struct ci_tx_queue *txq, 3279 uint32_t free_cnt) 3280 { 3281 int i, n, cnt; 3282 3283 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 3284 free_cnt = txq->nb_tx_desc; 3285 3286 cnt = free_cnt - free_cnt % txq->tx_rs_thresh; 3287 3288 for (i = 0; i < cnt; i += n) { 3289 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh) 3290 break; 3291 3292 n = ice_tx_free_bufs(txq); 3293 3294 if (n == 0) 3295 break; 3296 } 3297 3298 return i; 3299 } 3300 3301 int 3302 ice_tx_done_cleanup(void *txq, uint32_t free_cnt) 3303 { 3304 struct ci_tx_queue *q = (struct ci_tx_queue *)txq; 3305 struct rte_eth_dev *dev = &rte_eth_devices[q->port_id]; 3306 struct ice_adapter *ad = 3307 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3308 3309 #ifdef RTE_ARCH_X86 3310 if (ad->tx_vec_allowed) 3311 return ice_tx_done_cleanup_vec(q, free_cnt); 3312 #endif 3313 if (ad->tx_simple_allowed) 3314 return ice_tx_done_cleanup_simple(q, free_cnt); 3315 else 3316 return ice_tx_done_cleanup_full(q, free_cnt); 3317 } 3318 3319 /* Populate 4 descriptors with data from 4 mbufs */ 3320 static inline void 3321 tx4(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkts) 3322 { 3323 uint64_t dma_addr; 3324 uint32_t i; 3325 3326 for (i = 0; i < 4; i++, txdp++, pkts++) { 3327 dma_addr = rte_mbuf_data_iova(*pkts); 3328 txdp->buf_addr = rte_cpu_to_le_64(dma_addr); 3329 txdp->cmd_type_offset_bsz = 3330 ice_build_ctob((uint32_t)ICE_TD_CMD, 0, 3331 (*pkts)->data_len, 0); 3332 } 3333 } 3334 3335 /* Populate 1 descriptor with data from 1 mbuf */ 3336 static inline void 3337 tx1(volatile struct ice_tx_desc *txdp, struct rte_mbuf **pkts) 3338 { 3339 uint64_t dma_addr; 3340 3341 dma_addr = rte_mbuf_data_iova(*pkts); 3342 txdp->buf_addr = rte_cpu_to_le_64(dma_addr); 3343 txdp->cmd_type_offset_bsz = 3344 ice_build_ctob((uint32_t)ICE_TD_CMD, 0, 3345 (*pkts)->data_len, 0); 3346 } 3347 3348 static inline void 3349 ice_tx_fill_hw_ring(struct ci_tx_queue *txq, struct rte_mbuf **pkts, 3350 uint16_t nb_pkts) 3351 { 3352 volatile struct ice_tx_desc *txdp = &txq->ice_tx_ring[txq->tx_tail]; 3353 struct ci_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 3354 const int N_PER_LOOP = 4; 3355 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 3356 int mainpart, leftover; 3357 int i, j; 3358 3359 /** 3360 * Process most of the packets in chunks of N pkts. Any 3361 * leftover packets will get processed one at a time. 3362 */ 3363 mainpart = nb_pkts & ((uint32_t)~N_PER_LOOP_MASK); 3364 leftover = nb_pkts & ((uint32_t)N_PER_LOOP_MASK); 3365 for (i = 0; i < mainpart; i += N_PER_LOOP) { 3366 /* Copy N mbuf pointers to the S/W ring */ 3367 for (j = 0; j < N_PER_LOOP; ++j) 3368 (txep + i + j)->mbuf = *(pkts + i + j); 3369 tx4(txdp + i, pkts + i); 3370 } 3371 3372 if (unlikely(leftover > 0)) { 3373 for (i = 0; i < leftover; ++i) { 3374 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 3375 tx1(txdp + mainpart + i, pkts + mainpart + i); 3376 } 3377 } 3378 } 3379 3380 static inline uint16_t 3381 tx_xmit_pkts(struct ci_tx_queue *txq, 3382 struct rte_mbuf **tx_pkts, 3383 uint16_t nb_pkts) 3384 { 3385 volatile struct ice_tx_desc *txr = txq->ice_tx_ring; 3386 uint16_t n = 0; 3387 3388 /** 3389 * Begin scanning the H/W ring for done descriptors when the number 3390 * of available descriptors drops below tx_free_thresh. For each done 3391 * descriptor, free the associated buffer. 3392 */ 3393 if (txq->nb_tx_free < txq->tx_free_thresh) 3394 ice_tx_free_bufs(txq); 3395 3396 /* Use available descriptor only */ 3397 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 3398 if (unlikely(!nb_pkts)) 3399 return 0; 3400 3401 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 3402 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 3403 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 3404 ice_tx_fill_hw_ring(txq, tx_pkts, n); 3405 txr[txq->tx_next_rs].cmd_type_offset_bsz |= 3406 rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) << 3407 ICE_TXD_QW1_CMD_S); 3408 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 3409 txq->tx_tail = 0; 3410 } 3411 3412 /* Fill hardware descriptor ring with mbuf data */ 3413 ice_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 3414 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 3415 3416 /* Determine if RS bit needs to be set */ 3417 if (txq->tx_tail > txq->tx_next_rs) { 3418 txr[txq->tx_next_rs].cmd_type_offset_bsz |= 3419 rte_cpu_to_le_64(((uint64_t)ICE_TX_DESC_CMD_RS) << 3420 ICE_TXD_QW1_CMD_S); 3421 txq->tx_next_rs = 3422 (uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh); 3423 if (txq->tx_next_rs >= txq->nb_tx_desc) 3424 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 3425 } 3426 3427 if (txq->tx_tail >= txq->nb_tx_desc) 3428 txq->tx_tail = 0; 3429 3430 /* Update the tx tail register */ 3431 ICE_PCI_REG_WC_WRITE(txq->qtx_tail, txq->tx_tail); 3432 3433 return nb_pkts; 3434 } 3435 3436 static uint16_t 3437 ice_xmit_pkts_simple(void *tx_queue, 3438 struct rte_mbuf **tx_pkts, 3439 uint16_t nb_pkts) 3440 { 3441 uint16_t nb_tx = 0; 3442 3443 if (likely(nb_pkts <= ICE_TX_MAX_BURST)) 3444 return tx_xmit_pkts((struct ci_tx_queue *)tx_queue, 3445 tx_pkts, nb_pkts); 3446 3447 while (nb_pkts) { 3448 uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts, 3449 ICE_TX_MAX_BURST); 3450 3451 ret = tx_xmit_pkts((struct ci_tx_queue *)tx_queue, 3452 &tx_pkts[nb_tx], num); 3453 nb_tx = (uint16_t)(nb_tx + ret); 3454 nb_pkts = (uint16_t)(nb_pkts - ret); 3455 if (ret < num) 3456 break; 3457 } 3458 3459 return nb_tx; 3460 } 3461 3462 void __rte_cold 3463 ice_set_rx_function(struct rte_eth_dev *dev) 3464 { 3465 PMD_INIT_FUNC_TRACE(); 3466 struct ice_adapter *ad = 3467 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3468 #ifdef RTE_ARCH_X86 3469 struct ice_rx_queue *rxq; 3470 int i; 3471 int rx_check_ret = -1; 3472 3473 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3474 ad->rx_use_avx512 = false; 3475 ad->rx_use_avx2 = false; 3476 rx_check_ret = ice_rx_vec_dev_check(dev); 3477 if (ad->ptp_ena) 3478 rx_check_ret = -1; 3479 ad->rx_vec_offload_support = 3480 (rx_check_ret == ICE_VECTOR_OFFLOAD_PATH); 3481 if (rx_check_ret >= 0 && ad->rx_bulk_alloc_allowed && 3482 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 3483 ad->rx_vec_allowed = true; 3484 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3485 rxq = dev->data->rx_queues[i]; 3486 if (rxq && ice_rxq_vec_setup(rxq)) { 3487 ad->rx_vec_allowed = false; 3488 break; 3489 } 3490 } 3491 3492 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 && 3493 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && 3494 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) 3495 #ifdef CC_AVX512_SUPPORT 3496 ad->rx_use_avx512 = true; 3497 #else 3498 PMD_DRV_LOG(NOTICE, 3499 "AVX512 is not supported in build env"); 3500 #endif 3501 if (!ad->rx_use_avx512 && 3502 (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 || 3503 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) && 3504 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256) 3505 ad->rx_use_avx2 = true; 3506 3507 } else { 3508 ad->rx_vec_allowed = false; 3509 } 3510 } 3511 3512 if (ad->rx_vec_allowed) { 3513 if (dev->data->scattered_rx) { 3514 if (ad->rx_use_avx512) { 3515 #ifdef CC_AVX512_SUPPORT 3516 if (ad->rx_vec_offload_support) { 3517 PMD_DRV_LOG(NOTICE, 3518 "Using AVX512 OFFLOAD Vector Scattered Rx (port %d).", 3519 dev->data->port_id); 3520 dev->rx_pkt_burst = 3521 ice_recv_scattered_pkts_vec_avx512_offload; 3522 } else { 3523 PMD_DRV_LOG(NOTICE, 3524 "Using AVX512 Vector Scattered Rx (port %d).", 3525 dev->data->port_id); 3526 dev->rx_pkt_burst = 3527 ice_recv_scattered_pkts_vec_avx512; 3528 } 3529 #endif 3530 } else if (ad->rx_use_avx2) { 3531 if (ad->rx_vec_offload_support) { 3532 PMD_DRV_LOG(NOTICE, 3533 "Using AVX2 OFFLOAD Vector Scattered Rx (port %d).", 3534 dev->data->port_id); 3535 dev->rx_pkt_burst = 3536 ice_recv_scattered_pkts_vec_avx2_offload; 3537 } else { 3538 PMD_DRV_LOG(NOTICE, 3539 "Using AVX2 Vector Scattered Rx (port %d).", 3540 dev->data->port_id); 3541 dev->rx_pkt_burst = 3542 ice_recv_scattered_pkts_vec_avx2; 3543 } 3544 } else { 3545 PMD_DRV_LOG(DEBUG, 3546 "Using Vector Scattered Rx (port %d).", 3547 dev->data->port_id); 3548 dev->rx_pkt_burst = ice_recv_scattered_pkts_vec; 3549 } 3550 } else { 3551 if (ad->rx_use_avx512) { 3552 #ifdef CC_AVX512_SUPPORT 3553 if (ad->rx_vec_offload_support) { 3554 PMD_DRV_LOG(NOTICE, 3555 "Using AVX512 OFFLOAD Vector Rx (port %d).", 3556 dev->data->port_id); 3557 dev->rx_pkt_burst = 3558 ice_recv_pkts_vec_avx512_offload; 3559 } else { 3560 PMD_DRV_LOG(NOTICE, 3561 "Using AVX512 Vector Rx (port %d).", 3562 dev->data->port_id); 3563 dev->rx_pkt_burst = 3564 ice_recv_pkts_vec_avx512; 3565 } 3566 #endif 3567 } else if (ad->rx_use_avx2) { 3568 if (ad->rx_vec_offload_support) { 3569 PMD_DRV_LOG(NOTICE, 3570 "Using AVX2 OFFLOAD Vector Rx (port %d).", 3571 dev->data->port_id); 3572 dev->rx_pkt_burst = 3573 ice_recv_pkts_vec_avx2_offload; 3574 } else { 3575 PMD_DRV_LOG(NOTICE, 3576 "Using AVX2 Vector Rx (port %d).", 3577 dev->data->port_id); 3578 dev->rx_pkt_burst = 3579 ice_recv_pkts_vec_avx2; 3580 } 3581 } else { 3582 PMD_DRV_LOG(DEBUG, 3583 "Using Vector Rx (port %d).", 3584 dev->data->port_id); 3585 dev->rx_pkt_burst = ice_recv_pkts_vec; 3586 } 3587 } 3588 return; 3589 } 3590 3591 #endif 3592 3593 if (dev->data->scattered_rx) { 3594 /* Set the non-LRO scattered function */ 3595 PMD_INIT_LOG(DEBUG, 3596 "Using a Scattered function on port %d.", 3597 dev->data->port_id); 3598 dev->rx_pkt_burst = ice_recv_scattered_pkts; 3599 } else if (ad->rx_bulk_alloc_allowed) { 3600 PMD_INIT_LOG(DEBUG, 3601 "Rx Burst Bulk Alloc Preconditions are " 3602 "satisfied. Rx Burst Bulk Alloc function " 3603 "will be used on port %d.", 3604 dev->data->port_id); 3605 dev->rx_pkt_burst = ice_recv_pkts_bulk_alloc; 3606 } else { 3607 PMD_INIT_LOG(DEBUG, 3608 "Rx Burst Bulk Alloc Preconditions are not " 3609 "satisfied, Normal Rx will be used on port %d.", 3610 dev->data->port_id); 3611 dev->rx_pkt_burst = ice_recv_pkts; 3612 } 3613 } 3614 3615 static const struct { 3616 eth_rx_burst_t pkt_burst; 3617 const char *info; 3618 } ice_rx_burst_infos[] = { 3619 { ice_recv_scattered_pkts, "Scalar Scattered" }, 3620 { ice_recv_pkts_bulk_alloc, "Scalar Bulk Alloc" }, 3621 { ice_recv_pkts, "Scalar" }, 3622 #ifdef RTE_ARCH_X86 3623 #ifdef CC_AVX512_SUPPORT 3624 { ice_recv_scattered_pkts_vec_avx512, "Vector AVX512 Scattered" }, 3625 { ice_recv_scattered_pkts_vec_avx512_offload, "Offload Vector AVX512 Scattered" }, 3626 { ice_recv_pkts_vec_avx512, "Vector AVX512" }, 3627 { ice_recv_pkts_vec_avx512_offload, "Offload Vector AVX512" }, 3628 #endif 3629 { ice_recv_scattered_pkts_vec_avx2, "Vector AVX2 Scattered" }, 3630 { ice_recv_scattered_pkts_vec_avx2_offload, "Offload Vector AVX2 Scattered" }, 3631 { ice_recv_pkts_vec_avx2, "Vector AVX2" }, 3632 { ice_recv_pkts_vec_avx2_offload, "Offload Vector AVX2" }, 3633 { ice_recv_scattered_pkts_vec, "Vector SSE Scattered" }, 3634 { ice_recv_pkts_vec, "Vector SSE" }, 3635 #endif 3636 }; 3637 3638 int 3639 ice_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 3640 struct rte_eth_burst_mode *mode) 3641 { 3642 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 3643 int ret = -EINVAL; 3644 unsigned int i; 3645 3646 for (i = 0; i < RTE_DIM(ice_rx_burst_infos); ++i) { 3647 if (pkt_burst == ice_rx_burst_infos[i].pkt_burst) { 3648 snprintf(mode->info, sizeof(mode->info), "%s", 3649 ice_rx_burst_infos[i].info); 3650 ret = 0; 3651 break; 3652 } 3653 } 3654 3655 return ret; 3656 } 3657 3658 void __rte_cold 3659 ice_set_tx_function_flag(struct rte_eth_dev *dev, struct ci_tx_queue *txq) 3660 { 3661 struct ice_adapter *ad = 3662 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3663 3664 /* Use a simple Tx queue if possible (only fast free is allowed) */ 3665 ad->tx_simple_allowed = 3666 (txq->offloads == 3667 (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) && 3668 txq->tx_rs_thresh >= ICE_TX_MAX_BURST); 3669 3670 if (ad->tx_simple_allowed) 3671 PMD_INIT_LOG(DEBUG, "Simple Tx can be enabled on Tx queue %u.", 3672 txq->queue_id); 3673 else 3674 PMD_INIT_LOG(DEBUG, 3675 "Simple Tx can NOT be enabled on Tx queue %u.", 3676 txq->queue_id); 3677 } 3678 3679 /********************************************************************* 3680 * 3681 * TX prep functions 3682 * 3683 **********************************************************************/ 3684 /* The default values of TSO MSS */ 3685 #define ICE_MIN_TSO_MSS 64 3686 #define ICE_MAX_TSO_MSS 9728 3687 #define ICE_MAX_TSO_FRAME_SIZE 262144 3688 3689 /*Check for empty mbuf*/ 3690 static inline uint16_t 3691 ice_check_empty_mbuf(struct rte_mbuf *tx_pkt) 3692 { 3693 struct rte_mbuf *txd = tx_pkt; 3694 3695 while (txd != NULL) { 3696 if (txd->data_len == 0) 3697 return -1; 3698 txd = txd->next; 3699 } 3700 3701 return 0; 3702 } 3703 3704 /* Tx mbuf check */ 3705 static uint16_t 3706 ice_xmit_pkts_check(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 3707 { 3708 struct ci_tx_queue *txq = tx_queue; 3709 uint16_t idx; 3710 struct rte_mbuf *mb; 3711 bool pkt_error = false; 3712 uint16_t good_pkts = nb_pkts; 3713 const char *reason = NULL; 3714 struct ice_adapter *adapter = txq->ice_vsi->adapter; 3715 uint64_t ol_flags; 3716 3717 for (idx = 0; idx < nb_pkts; idx++) { 3718 mb = tx_pkts[idx]; 3719 ol_flags = mb->ol_flags; 3720 3721 if ((adapter->devargs.mbuf_check & ICE_MBUF_CHECK_F_TX_MBUF) && 3722 (rte_mbuf_check(mb, 1, &reason) != 0)) { 3723 PMD_TX_LOG(ERR, "INVALID mbuf: %s", reason); 3724 pkt_error = true; 3725 break; 3726 } 3727 3728 if ((adapter->devargs.mbuf_check & ICE_MBUF_CHECK_F_TX_SIZE) && 3729 (mb->data_len > mb->pkt_len || 3730 mb->data_len < ICE_TX_MIN_PKT_LEN || 3731 mb->data_len > ICE_FRAME_SIZE_MAX)) { 3732 PMD_TX_LOG(ERR, "INVALID mbuf: data_len (%u) is out of range, reasonable range (%d - %d)", 3733 mb->data_len, ICE_TX_MIN_PKT_LEN, ICE_FRAME_SIZE_MAX); 3734 pkt_error = true; 3735 break; 3736 } 3737 3738 if (adapter->devargs.mbuf_check & ICE_MBUF_CHECK_F_TX_SEGMENT) { 3739 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { 3740 /** 3741 * No TSO case: nb->segs, pkt_len to not exceed 3742 * the limites. 3743 */ 3744 if (mb->nb_segs > ICE_TX_MTU_SEG_MAX) { 3745 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs (%d) exceeds HW limit, maximum allowed value is %d", 3746 mb->nb_segs, ICE_TX_MTU_SEG_MAX); 3747 pkt_error = true; 3748 break; 3749 } 3750 if (mb->pkt_len > ICE_FRAME_SIZE_MAX) { 3751 PMD_TX_LOG(ERR, "INVALID mbuf: pkt_len (%d) exceeds HW limit, maximum allowed value is %d", 3752 mb->nb_segs, ICE_FRAME_SIZE_MAX); 3753 pkt_error = true; 3754 break; 3755 } 3756 } else if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 3757 /** TSO case: tso_segsz, nb_segs, pkt_len not exceed 3758 * the limits. 3759 */ 3760 if (mb->tso_segsz < ICE_MIN_TSO_MSS || 3761 mb->tso_segsz > ICE_MAX_TSO_MSS) { 3762 /** 3763 * MSS outside the range are considered malicious 3764 */ 3765 PMD_TX_LOG(ERR, "INVALID mbuf: tso_segsz (%u) is out of range, reasonable range (%d - %u)", 3766 mb->tso_segsz, ICE_MIN_TSO_MSS, ICE_MAX_TSO_MSS); 3767 pkt_error = true; 3768 break; 3769 } 3770 if (mb->nb_segs > ((struct ci_tx_queue *)tx_queue)->nb_tx_desc) { 3771 PMD_TX_LOG(ERR, "INVALID mbuf: nb_segs out of ring length"); 3772 pkt_error = true; 3773 break; 3774 } 3775 } 3776 } 3777 3778 if (adapter->devargs.mbuf_check & ICE_MBUF_CHECK_F_TX_OFFLOAD) { 3779 if (ol_flags & ICE_TX_OFFLOAD_NOTSUP_MASK) { 3780 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload is not supported"); 3781 pkt_error = true; 3782 break; 3783 } 3784 3785 if (!rte_validate_tx_offload(mb)) { 3786 PMD_TX_LOG(ERR, "INVALID mbuf: TX offload setup error"); 3787 pkt_error = true; 3788 break; 3789 } 3790 } 3791 } 3792 3793 if (pkt_error) { 3794 txq->mbuf_errors++; 3795 good_pkts = idx; 3796 if (good_pkts == 0) 3797 return 0; 3798 } 3799 3800 return adapter->tx_pkt_burst(tx_queue, tx_pkts, good_pkts); 3801 } 3802 3803 uint16_t 3804 ice_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 3805 uint16_t nb_pkts) 3806 { 3807 int i, ret; 3808 uint64_t ol_flags; 3809 struct rte_mbuf *m; 3810 3811 for (i = 0; i < nb_pkts; i++) { 3812 m = tx_pkts[i]; 3813 ol_flags = m->ol_flags; 3814 3815 if (!(ol_flags & RTE_MBUF_F_TX_TCP_SEG) && 3816 /** 3817 * No TSO case: nb->segs, pkt_len to not exceed 3818 * the limites. 3819 */ 3820 (m->nb_segs > ICE_TX_MTU_SEG_MAX || 3821 m->pkt_len > ICE_FRAME_SIZE_MAX)) { 3822 rte_errno = EINVAL; 3823 return i; 3824 } else if (ol_flags & RTE_MBUF_F_TX_TCP_SEG && 3825 /** TSO case: tso_segsz, nb_segs, pkt_len not exceed 3826 * the limits. 3827 */ 3828 (m->tso_segsz < ICE_MIN_TSO_MSS || 3829 m->tso_segsz > ICE_MAX_TSO_MSS || 3830 m->nb_segs > 3831 ((struct ci_tx_queue *)tx_queue)->nb_tx_desc || 3832 m->pkt_len > ICE_MAX_TSO_FRAME_SIZE)) { 3833 /** 3834 * MSS outside the range are considered malicious 3835 */ 3836 rte_errno = EINVAL; 3837 return i; 3838 } 3839 3840 if (m->pkt_len < ICE_TX_MIN_PKT_LEN) { 3841 rte_errno = EINVAL; 3842 return i; 3843 } 3844 3845 #ifdef RTE_ETHDEV_DEBUG_TX 3846 ret = rte_validate_tx_offload(m); 3847 if (ret != 0) { 3848 rte_errno = -ret; 3849 return i; 3850 } 3851 #endif 3852 ret = rte_net_intel_cksum_prepare(m); 3853 if (ret != 0) { 3854 rte_errno = -ret; 3855 return i; 3856 } 3857 3858 if (ice_check_empty_mbuf(m) != 0) { 3859 rte_errno = EINVAL; 3860 return i; 3861 } 3862 } 3863 return i; 3864 } 3865 3866 void __rte_cold 3867 ice_set_tx_function(struct rte_eth_dev *dev) 3868 { 3869 struct ice_adapter *ad = 3870 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 3871 int mbuf_check = ad->devargs.mbuf_check; 3872 #ifdef RTE_ARCH_X86 3873 struct ci_tx_queue *txq; 3874 int i; 3875 int tx_check_ret = -1; 3876 3877 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 3878 ad->tx_use_avx2 = false; 3879 ad->tx_use_avx512 = false; 3880 tx_check_ret = ice_tx_vec_dev_check(dev); 3881 if (tx_check_ret >= 0 && 3882 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 3883 ad->tx_vec_allowed = true; 3884 3885 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_512 && 3886 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1 && 3887 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) == 1) 3888 #ifdef CC_AVX512_SUPPORT 3889 ad->tx_use_avx512 = true; 3890 #else 3891 PMD_DRV_LOG(NOTICE, 3892 "AVX512 is not supported in build env"); 3893 #endif 3894 if (!ad->tx_use_avx512 && 3895 (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 || 3896 rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) && 3897 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256) 3898 ad->tx_use_avx2 = true; 3899 3900 if (!ad->tx_use_avx2 && !ad->tx_use_avx512 && 3901 tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) 3902 ad->tx_vec_allowed = false; 3903 3904 if (ad->tx_vec_allowed) { 3905 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3906 txq = dev->data->tx_queues[i]; 3907 if (txq && ice_txq_vec_setup(txq)) { 3908 ad->tx_vec_allowed = false; 3909 break; 3910 } 3911 } 3912 } 3913 } else { 3914 ad->tx_vec_allowed = false; 3915 } 3916 } 3917 3918 if (ad->tx_vec_allowed) { 3919 dev->tx_pkt_prepare = NULL; 3920 if (ad->tx_use_avx512) { 3921 #ifdef CC_AVX512_SUPPORT 3922 if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) { 3923 PMD_DRV_LOG(NOTICE, 3924 "Using AVX512 OFFLOAD Vector Tx (port %d).", 3925 dev->data->port_id); 3926 dev->tx_pkt_burst = 3927 ice_xmit_pkts_vec_avx512_offload; 3928 dev->tx_pkt_prepare = ice_prep_pkts; 3929 } else { 3930 PMD_DRV_LOG(NOTICE, 3931 "Using AVX512 Vector Tx (port %d).", 3932 dev->data->port_id); 3933 dev->tx_pkt_burst = ice_xmit_pkts_vec_avx512; 3934 } 3935 #endif 3936 } else { 3937 if (tx_check_ret == ICE_VECTOR_OFFLOAD_PATH) { 3938 PMD_DRV_LOG(NOTICE, 3939 "Using AVX2 OFFLOAD Vector Tx (port %d).", 3940 dev->data->port_id); 3941 dev->tx_pkt_burst = 3942 ice_xmit_pkts_vec_avx2_offload; 3943 dev->tx_pkt_prepare = ice_prep_pkts; 3944 } else { 3945 PMD_DRV_LOG(DEBUG, "Using %sVector Tx (port %d).", 3946 ad->tx_use_avx2 ? "avx2 " : "", 3947 dev->data->port_id); 3948 dev->tx_pkt_burst = ad->tx_use_avx2 ? 3949 ice_xmit_pkts_vec_avx2 : 3950 ice_xmit_pkts_vec; 3951 } 3952 } 3953 3954 if (mbuf_check) { 3955 ad->tx_pkt_burst = dev->tx_pkt_burst; 3956 dev->tx_pkt_burst = ice_xmit_pkts_check; 3957 } 3958 return; 3959 } 3960 #endif 3961 3962 if (ad->tx_simple_allowed) { 3963 PMD_INIT_LOG(DEBUG, "Simple tx finally be used."); 3964 dev->tx_pkt_burst = ice_xmit_pkts_simple; 3965 dev->tx_pkt_prepare = NULL; 3966 } else { 3967 PMD_INIT_LOG(DEBUG, "Normal tx finally be used."); 3968 dev->tx_pkt_burst = ice_xmit_pkts; 3969 dev->tx_pkt_prepare = ice_prep_pkts; 3970 } 3971 3972 if (mbuf_check) { 3973 ad->tx_pkt_burst = dev->tx_pkt_burst; 3974 dev->tx_pkt_burst = ice_xmit_pkts_check; 3975 } 3976 } 3977 3978 static const struct { 3979 eth_tx_burst_t pkt_burst; 3980 const char *info; 3981 } ice_tx_burst_infos[] = { 3982 { ice_xmit_pkts_simple, "Scalar Simple" }, 3983 { ice_xmit_pkts, "Scalar" }, 3984 #ifdef RTE_ARCH_X86 3985 #ifdef CC_AVX512_SUPPORT 3986 { ice_xmit_pkts_vec_avx512, "Vector AVX512" }, 3987 { ice_xmit_pkts_vec_avx512_offload, "Offload Vector AVX512" }, 3988 #endif 3989 { ice_xmit_pkts_vec_avx2, "Vector AVX2" }, 3990 { ice_xmit_pkts_vec_avx2_offload, "Offload Vector AVX2" }, 3991 { ice_xmit_pkts_vec, "Vector SSE" }, 3992 #endif 3993 }; 3994 3995 int 3996 ice_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 3997 struct rte_eth_burst_mode *mode) 3998 { 3999 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 4000 int ret = -EINVAL; 4001 unsigned int i; 4002 4003 for (i = 0; i < RTE_DIM(ice_tx_burst_infos); ++i) { 4004 if (pkt_burst == ice_tx_burst_infos[i].pkt_burst) { 4005 snprintf(mode->info, sizeof(mode->info), "%s", 4006 ice_tx_burst_infos[i].info); 4007 ret = 0; 4008 break; 4009 } 4010 } 4011 4012 return ret; 4013 } 4014 4015 /* For each value it means, datasheet of hardware can tell more details 4016 * 4017 * @note: fix ice_dev_supported_ptypes_get() if any change here. 4018 */ 4019 static inline uint32_t 4020 ice_get_default_pkt_type(uint16_t ptype) 4021 { 4022 static const alignas(RTE_CACHE_LINE_SIZE) uint32_t type_table[ICE_MAX_PKT_TYPE] = { 4023 /* L2 types */ 4024 /* [0] reserved */ 4025 [1] = RTE_PTYPE_L2_ETHER, 4026 [2] = RTE_PTYPE_L2_ETHER_TIMESYNC, 4027 /* [3] - [5] reserved */ 4028 [6] = RTE_PTYPE_L2_ETHER_LLDP, 4029 /* [7] - [10] reserved */ 4030 [11] = RTE_PTYPE_L2_ETHER_ARP, 4031 /* [12] - [21] reserved */ 4032 4033 /* Non tunneled IPv4 */ 4034 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4035 RTE_PTYPE_L4_FRAG, 4036 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4037 RTE_PTYPE_L4_NONFRAG, 4038 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4039 RTE_PTYPE_L4_UDP, 4040 /* [25] reserved */ 4041 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4042 RTE_PTYPE_L4_TCP, 4043 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4044 RTE_PTYPE_L4_SCTP, 4045 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4046 RTE_PTYPE_L4_ICMP, 4047 4048 /* IPv4 --> IPv4 */ 4049 [29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4050 RTE_PTYPE_TUNNEL_IP | 4051 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4052 RTE_PTYPE_INNER_L4_FRAG, 4053 [30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4054 RTE_PTYPE_TUNNEL_IP | 4055 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4056 RTE_PTYPE_INNER_L4_NONFRAG, 4057 [31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4058 RTE_PTYPE_TUNNEL_IP | 4059 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4060 RTE_PTYPE_INNER_L4_UDP, 4061 /* [32] reserved */ 4062 [33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4063 RTE_PTYPE_TUNNEL_IP | 4064 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4065 RTE_PTYPE_INNER_L4_TCP, 4066 [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4067 RTE_PTYPE_TUNNEL_IP | 4068 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4069 RTE_PTYPE_INNER_L4_SCTP, 4070 [35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4071 RTE_PTYPE_TUNNEL_IP | 4072 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4073 RTE_PTYPE_INNER_L4_ICMP, 4074 4075 /* IPv4 --> IPv6 */ 4076 [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4077 RTE_PTYPE_TUNNEL_IP | 4078 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4079 RTE_PTYPE_INNER_L4_FRAG, 4080 [37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4081 RTE_PTYPE_TUNNEL_IP | 4082 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4083 RTE_PTYPE_INNER_L4_NONFRAG, 4084 [38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4085 RTE_PTYPE_TUNNEL_IP | 4086 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4087 RTE_PTYPE_INNER_L4_UDP, 4088 /* [39] reserved */ 4089 [40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4090 RTE_PTYPE_TUNNEL_IP | 4091 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4092 RTE_PTYPE_INNER_L4_TCP, 4093 [41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4094 RTE_PTYPE_TUNNEL_IP | 4095 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4096 RTE_PTYPE_INNER_L4_SCTP, 4097 [42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4098 RTE_PTYPE_TUNNEL_IP | 4099 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4100 RTE_PTYPE_INNER_L4_ICMP, 4101 4102 /* IPv4 --> GRE/Teredo/VXLAN */ 4103 [43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4104 RTE_PTYPE_TUNNEL_GRENAT, 4105 4106 /* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */ 4107 [44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4108 RTE_PTYPE_TUNNEL_GRENAT | 4109 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4110 RTE_PTYPE_INNER_L4_FRAG, 4111 [45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4112 RTE_PTYPE_TUNNEL_GRENAT | 4113 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4114 RTE_PTYPE_INNER_L4_NONFRAG, 4115 [46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4116 RTE_PTYPE_TUNNEL_GRENAT | 4117 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4118 RTE_PTYPE_INNER_L4_UDP, 4119 /* [47] reserved */ 4120 [48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4121 RTE_PTYPE_TUNNEL_GRENAT | 4122 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4123 RTE_PTYPE_INNER_L4_TCP, 4124 [49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4125 RTE_PTYPE_TUNNEL_GRENAT | 4126 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4127 RTE_PTYPE_INNER_L4_SCTP, 4128 [50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4129 RTE_PTYPE_TUNNEL_GRENAT | 4130 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4131 RTE_PTYPE_INNER_L4_ICMP, 4132 4133 /* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */ 4134 [51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4135 RTE_PTYPE_TUNNEL_GRENAT | 4136 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4137 RTE_PTYPE_INNER_L4_FRAG, 4138 [52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4139 RTE_PTYPE_TUNNEL_GRENAT | 4140 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4141 RTE_PTYPE_INNER_L4_NONFRAG, 4142 [53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4143 RTE_PTYPE_TUNNEL_GRENAT | 4144 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4145 RTE_PTYPE_INNER_L4_UDP, 4146 /* [54] reserved */ 4147 [55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4148 RTE_PTYPE_TUNNEL_GRENAT | 4149 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4150 RTE_PTYPE_INNER_L4_TCP, 4151 [56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4152 RTE_PTYPE_TUNNEL_GRENAT | 4153 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4154 RTE_PTYPE_INNER_L4_SCTP, 4155 [57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4156 RTE_PTYPE_TUNNEL_GRENAT | 4157 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4158 RTE_PTYPE_INNER_L4_ICMP, 4159 4160 /* IPv4 --> GRE/Teredo/VXLAN --> MAC */ 4161 [58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4162 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, 4163 4164 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ 4165 [59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4166 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4167 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4168 RTE_PTYPE_INNER_L4_FRAG, 4169 [60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4170 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4171 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4172 RTE_PTYPE_INNER_L4_NONFRAG, 4173 [61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4174 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4175 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4176 RTE_PTYPE_INNER_L4_UDP, 4177 /* [62] reserved */ 4178 [63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4179 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4180 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4181 RTE_PTYPE_INNER_L4_TCP, 4182 [64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4183 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4184 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4185 RTE_PTYPE_INNER_L4_SCTP, 4186 [65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4187 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4188 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4189 RTE_PTYPE_INNER_L4_ICMP, 4190 4191 /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ 4192 [66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4193 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4194 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4195 RTE_PTYPE_INNER_L4_FRAG, 4196 [67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4197 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4198 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4199 RTE_PTYPE_INNER_L4_NONFRAG, 4200 [68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4201 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4202 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4203 RTE_PTYPE_INNER_L4_UDP, 4204 /* [69] reserved */ 4205 [70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4206 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4207 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4208 RTE_PTYPE_INNER_L4_TCP, 4209 [71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4210 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4211 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4212 RTE_PTYPE_INNER_L4_SCTP, 4213 [72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4214 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4215 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4216 RTE_PTYPE_INNER_L4_ICMP, 4217 /* [73] - [87] reserved */ 4218 4219 /* Non tunneled IPv6 */ 4220 [88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4221 RTE_PTYPE_L4_FRAG, 4222 [89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4223 RTE_PTYPE_L4_NONFRAG, 4224 [90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4225 RTE_PTYPE_L4_UDP, 4226 /* [91] reserved */ 4227 [92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4228 RTE_PTYPE_L4_TCP, 4229 [93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4230 RTE_PTYPE_L4_SCTP, 4231 [94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4232 RTE_PTYPE_L4_ICMP, 4233 4234 /* IPv6 --> IPv4 */ 4235 [95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4236 RTE_PTYPE_TUNNEL_IP | 4237 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4238 RTE_PTYPE_INNER_L4_FRAG, 4239 [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4240 RTE_PTYPE_TUNNEL_IP | 4241 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4242 RTE_PTYPE_INNER_L4_NONFRAG, 4243 [97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4244 RTE_PTYPE_TUNNEL_IP | 4245 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4246 RTE_PTYPE_INNER_L4_UDP, 4247 /* [98] reserved */ 4248 [99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4249 RTE_PTYPE_TUNNEL_IP | 4250 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4251 RTE_PTYPE_INNER_L4_TCP, 4252 [100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4253 RTE_PTYPE_TUNNEL_IP | 4254 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4255 RTE_PTYPE_INNER_L4_SCTP, 4256 [101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4257 RTE_PTYPE_TUNNEL_IP | 4258 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4259 RTE_PTYPE_INNER_L4_ICMP, 4260 4261 /* IPv6 --> IPv6 */ 4262 [102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4263 RTE_PTYPE_TUNNEL_IP | 4264 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4265 RTE_PTYPE_INNER_L4_FRAG, 4266 [103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4267 RTE_PTYPE_TUNNEL_IP | 4268 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4269 RTE_PTYPE_INNER_L4_NONFRAG, 4270 [104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4271 RTE_PTYPE_TUNNEL_IP | 4272 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4273 RTE_PTYPE_INNER_L4_UDP, 4274 /* [105] reserved */ 4275 [106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4276 RTE_PTYPE_TUNNEL_IP | 4277 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4278 RTE_PTYPE_INNER_L4_TCP, 4279 [107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4280 RTE_PTYPE_TUNNEL_IP | 4281 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4282 RTE_PTYPE_INNER_L4_SCTP, 4283 [108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4284 RTE_PTYPE_TUNNEL_IP | 4285 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4286 RTE_PTYPE_INNER_L4_ICMP, 4287 4288 /* IPv6 --> GRE/Teredo/VXLAN */ 4289 [109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4290 RTE_PTYPE_TUNNEL_GRENAT, 4291 4292 /* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */ 4293 [110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4294 RTE_PTYPE_TUNNEL_GRENAT | 4295 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4296 RTE_PTYPE_INNER_L4_FRAG, 4297 [111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4298 RTE_PTYPE_TUNNEL_GRENAT | 4299 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4300 RTE_PTYPE_INNER_L4_NONFRAG, 4301 [112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4302 RTE_PTYPE_TUNNEL_GRENAT | 4303 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4304 RTE_PTYPE_INNER_L4_UDP, 4305 /* [113] reserved */ 4306 [114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4307 RTE_PTYPE_TUNNEL_GRENAT | 4308 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4309 RTE_PTYPE_INNER_L4_TCP, 4310 [115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4311 RTE_PTYPE_TUNNEL_GRENAT | 4312 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4313 RTE_PTYPE_INNER_L4_SCTP, 4314 [116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4315 RTE_PTYPE_TUNNEL_GRENAT | 4316 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4317 RTE_PTYPE_INNER_L4_ICMP, 4318 4319 /* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */ 4320 [117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4321 RTE_PTYPE_TUNNEL_GRENAT | 4322 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4323 RTE_PTYPE_INNER_L4_FRAG, 4324 [118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4325 RTE_PTYPE_TUNNEL_GRENAT | 4326 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4327 RTE_PTYPE_INNER_L4_NONFRAG, 4328 [119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4329 RTE_PTYPE_TUNNEL_GRENAT | 4330 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4331 RTE_PTYPE_INNER_L4_UDP, 4332 /* [120] reserved */ 4333 [121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4334 RTE_PTYPE_TUNNEL_GRENAT | 4335 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4336 RTE_PTYPE_INNER_L4_TCP, 4337 [122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4338 RTE_PTYPE_TUNNEL_GRENAT | 4339 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4340 RTE_PTYPE_INNER_L4_SCTP, 4341 [123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4342 RTE_PTYPE_TUNNEL_GRENAT | 4343 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4344 RTE_PTYPE_INNER_L4_ICMP, 4345 4346 /* IPv6 --> GRE/Teredo/VXLAN --> MAC */ 4347 [124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4348 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER, 4349 4350 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */ 4351 [125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4352 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4353 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4354 RTE_PTYPE_INNER_L4_FRAG, 4355 [126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4356 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4357 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4358 RTE_PTYPE_INNER_L4_NONFRAG, 4359 [127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4360 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4361 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4362 RTE_PTYPE_INNER_L4_UDP, 4363 /* [128] reserved */ 4364 [129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4365 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4366 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4367 RTE_PTYPE_INNER_L4_TCP, 4368 [130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4369 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4370 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4371 RTE_PTYPE_INNER_L4_SCTP, 4372 [131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4373 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4374 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4375 RTE_PTYPE_INNER_L4_ICMP, 4376 4377 /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */ 4378 [132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4379 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4380 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4381 RTE_PTYPE_INNER_L4_FRAG, 4382 [133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4383 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4384 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4385 RTE_PTYPE_INNER_L4_NONFRAG, 4386 [134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4387 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4388 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4389 RTE_PTYPE_INNER_L4_UDP, 4390 /* [135] reserved */ 4391 [136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4392 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4393 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4394 RTE_PTYPE_INNER_L4_TCP, 4395 [137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4396 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4397 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4398 RTE_PTYPE_INNER_L4_SCTP, 4399 [138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4400 RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER | 4401 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4402 RTE_PTYPE_INNER_L4_ICMP, 4403 /* [139] - [299] reserved */ 4404 4405 /* PPPoE */ 4406 [300] = RTE_PTYPE_L2_ETHER_PPPOE, 4407 [301] = RTE_PTYPE_L2_ETHER_PPPOE, 4408 4409 /* PPPoE --> IPv4 */ 4410 [302] = RTE_PTYPE_L2_ETHER_PPPOE | 4411 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4412 RTE_PTYPE_L4_FRAG, 4413 [303] = RTE_PTYPE_L2_ETHER_PPPOE | 4414 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4415 RTE_PTYPE_L4_NONFRAG, 4416 [304] = RTE_PTYPE_L2_ETHER_PPPOE | 4417 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4418 RTE_PTYPE_L4_UDP, 4419 [305] = RTE_PTYPE_L2_ETHER_PPPOE | 4420 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4421 RTE_PTYPE_L4_TCP, 4422 [306] = RTE_PTYPE_L2_ETHER_PPPOE | 4423 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4424 RTE_PTYPE_L4_SCTP, 4425 [307] = RTE_PTYPE_L2_ETHER_PPPOE | 4426 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4427 RTE_PTYPE_L4_ICMP, 4428 4429 /* PPPoE --> IPv6 */ 4430 [308] = RTE_PTYPE_L2_ETHER_PPPOE | 4431 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4432 RTE_PTYPE_L4_FRAG, 4433 [309] = RTE_PTYPE_L2_ETHER_PPPOE | 4434 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4435 RTE_PTYPE_L4_NONFRAG, 4436 [310] = RTE_PTYPE_L2_ETHER_PPPOE | 4437 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4438 RTE_PTYPE_L4_UDP, 4439 [311] = RTE_PTYPE_L2_ETHER_PPPOE | 4440 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4441 RTE_PTYPE_L4_TCP, 4442 [312] = RTE_PTYPE_L2_ETHER_PPPOE | 4443 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4444 RTE_PTYPE_L4_SCTP, 4445 [313] = RTE_PTYPE_L2_ETHER_PPPOE | 4446 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4447 RTE_PTYPE_L4_ICMP, 4448 /* [314] - [324] reserved */ 4449 4450 /* IPv4/IPv6 --> GTPC/GTPU */ 4451 [325] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4452 RTE_PTYPE_TUNNEL_GTPC, 4453 [326] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4454 RTE_PTYPE_TUNNEL_GTPC, 4455 [327] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4456 RTE_PTYPE_TUNNEL_GTPC, 4457 [328] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4458 RTE_PTYPE_TUNNEL_GTPC, 4459 [329] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4460 RTE_PTYPE_TUNNEL_GTPU, 4461 [330] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4462 RTE_PTYPE_TUNNEL_GTPU, 4463 4464 /* IPv4 --> GTPU --> IPv4 */ 4465 [331] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4466 RTE_PTYPE_TUNNEL_GTPU | 4467 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4468 RTE_PTYPE_INNER_L4_FRAG, 4469 [332] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4470 RTE_PTYPE_TUNNEL_GTPU | 4471 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4472 RTE_PTYPE_INNER_L4_NONFRAG, 4473 [333] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4474 RTE_PTYPE_TUNNEL_GTPU | 4475 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4476 RTE_PTYPE_INNER_L4_UDP, 4477 [334] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4478 RTE_PTYPE_TUNNEL_GTPU | 4479 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4480 RTE_PTYPE_INNER_L4_TCP, 4481 [335] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4482 RTE_PTYPE_TUNNEL_GTPU | 4483 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4484 RTE_PTYPE_INNER_L4_ICMP, 4485 4486 /* IPv6 --> GTPU --> IPv4 */ 4487 [336] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4488 RTE_PTYPE_TUNNEL_GTPU | 4489 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4490 RTE_PTYPE_INNER_L4_FRAG, 4491 [337] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4492 RTE_PTYPE_TUNNEL_GTPU | 4493 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4494 RTE_PTYPE_INNER_L4_NONFRAG, 4495 [338] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4496 RTE_PTYPE_TUNNEL_GTPU | 4497 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4498 RTE_PTYPE_INNER_L4_UDP, 4499 [339] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4500 RTE_PTYPE_TUNNEL_GTPU | 4501 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4502 RTE_PTYPE_INNER_L4_TCP, 4503 [340] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4504 RTE_PTYPE_TUNNEL_GTPU | 4505 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 4506 RTE_PTYPE_INNER_L4_ICMP, 4507 4508 /* IPv4 --> GTPU --> IPv6 */ 4509 [341] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4510 RTE_PTYPE_TUNNEL_GTPU | 4511 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4512 RTE_PTYPE_INNER_L4_FRAG, 4513 [342] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4514 RTE_PTYPE_TUNNEL_GTPU | 4515 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4516 RTE_PTYPE_INNER_L4_NONFRAG, 4517 [343] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4518 RTE_PTYPE_TUNNEL_GTPU | 4519 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4520 RTE_PTYPE_INNER_L4_UDP, 4521 [344] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4522 RTE_PTYPE_TUNNEL_GTPU | 4523 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4524 RTE_PTYPE_INNER_L4_TCP, 4525 [345] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4526 RTE_PTYPE_TUNNEL_GTPU | 4527 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4528 RTE_PTYPE_INNER_L4_ICMP, 4529 4530 /* IPv6 --> GTPU --> IPv6 */ 4531 [346] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4532 RTE_PTYPE_TUNNEL_GTPU | 4533 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4534 RTE_PTYPE_INNER_L4_FRAG, 4535 [347] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4536 RTE_PTYPE_TUNNEL_GTPU | 4537 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4538 RTE_PTYPE_INNER_L4_NONFRAG, 4539 [348] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4540 RTE_PTYPE_TUNNEL_GTPU | 4541 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4542 RTE_PTYPE_INNER_L4_UDP, 4543 [349] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4544 RTE_PTYPE_TUNNEL_GTPU | 4545 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4546 RTE_PTYPE_INNER_L4_TCP, 4547 [350] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4548 RTE_PTYPE_TUNNEL_GTPU | 4549 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 4550 RTE_PTYPE_INNER_L4_ICMP, 4551 4552 /* IPv4 --> UDP ECPRI */ 4553 [372] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4554 RTE_PTYPE_L4_UDP, 4555 [373] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4556 RTE_PTYPE_L4_UDP, 4557 [374] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4558 RTE_PTYPE_L4_UDP, 4559 [375] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4560 RTE_PTYPE_L4_UDP, 4561 [376] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4562 RTE_PTYPE_L4_UDP, 4563 [377] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4564 RTE_PTYPE_L4_UDP, 4565 [378] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4566 RTE_PTYPE_L4_UDP, 4567 [379] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4568 RTE_PTYPE_L4_UDP, 4569 [380] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4570 RTE_PTYPE_L4_UDP, 4571 [381] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 4572 RTE_PTYPE_L4_UDP, 4573 4574 /* IPV6 --> UDP ECPRI */ 4575 [382] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4576 RTE_PTYPE_L4_UDP, 4577 [383] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4578 RTE_PTYPE_L4_UDP, 4579 [384] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4580 RTE_PTYPE_L4_UDP, 4581 [385] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4582 RTE_PTYPE_L4_UDP, 4583 [386] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4584 RTE_PTYPE_L4_UDP, 4585 [387] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4586 RTE_PTYPE_L4_UDP, 4587 [388] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4588 RTE_PTYPE_L4_UDP, 4589 [389] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4590 RTE_PTYPE_L4_UDP, 4591 [390] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4592 RTE_PTYPE_L4_UDP, 4593 [391] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 4594 RTE_PTYPE_L4_UDP, 4595 /* All others reserved */ 4596 }; 4597 4598 return type_table[ptype]; 4599 } 4600 4601 void __rte_cold 4602 ice_set_default_ptype_table(struct rte_eth_dev *dev) 4603 { 4604 struct ice_adapter *ad = 4605 ICE_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 4606 int i; 4607 4608 for (i = 0; i < ICE_MAX_PKT_TYPE; i++) 4609 ad->ptype_tbl[i] = ice_get_default_pkt_type(i); 4610 } 4611 4612 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S 1 4613 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_M \ 4614 (0x3UL << ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S) 4615 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_ADD 0 4616 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_DEL 0x1 4617 4618 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S 4 4619 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_M \ 4620 (1 << ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S) 4621 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S 5 4622 #define ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_M \ 4623 (1 << ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S) 4624 4625 /* 4626 * check the programming status descriptor in rx queue. 4627 * done after Programming Flow Director is programmed on 4628 * tx queue 4629 */ 4630 static inline int 4631 ice_check_fdir_programming_status(struct ice_rx_queue *rxq) 4632 { 4633 volatile union ice_32byte_rx_desc *rxdp; 4634 uint64_t qword1; 4635 uint32_t rx_status; 4636 uint32_t error; 4637 uint32_t id; 4638 int ret = -EAGAIN; 4639 4640 rxdp = (volatile union ice_32byte_rx_desc *) 4641 (&rxq->rx_ring[rxq->rx_tail]); 4642 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len); 4643 rx_status = (qword1 & ICE_RXD_QW1_STATUS_M) 4644 >> ICE_RXD_QW1_STATUS_S; 4645 4646 if (rx_status & (1 << ICE_RX_DESC_STATUS_DD_S)) { 4647 ret = 0; 4648 error = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_M) >> 4649 ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_S; 4650 id = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_M) >> 4651 ICE_RX_PROG_STATUS_DESC_WB_QW1_PROGID_S; 4652 if (error) { 4653 if (id == ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_ADD) 4654 PMD_DRV_LOG(ERR, "Failed to add FDIR rule."); 4655 else if (id == ICE_RX_PROG_STATUS_DESC_WB_QW1_PROG_DEL) 4656 PMD_DRV_LOG(ERR, "Failed to remove FDIR rule."); 4657 ret = -EINVAL; 4658 goto err; 4659 } 4660 error = (qword1 & ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_M) >> 4661 ICE_RX_PROG_STATUS_DESC_WB_QW1_FAIL_PROF_S; 4662 if (error) { 4663 PMD_DRV_LOG(ERR, "Failed to create FDIR profile."); 4664 ret = -EINVAL; 4665 } 4666 err: 4667 rxdp->wb.qword1.status_error_len = 0; 4668 rxq->rx_tail++; 4669 if (unlikely(rxq->rx_tail == rxq->nb_rx_desc)) 4670 rxq->rx_tail = 0; 4671 if (rxq->rx_tail == 0) 4672 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 4673 else 4674 ICE_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_tail - 1); 4675 } 4676 4677 return ret; 4678 } 4679 4680 #define ICE_FDIR_MAX_WAIT_US 10000 4681 4682 int 4683 ice_fdir_programming(struct ice_pf *pf, struct ice_fltr_desc *fdir_desc) 4684 { 4685 struct ci_tx_queue *txq = pf->fdir.txq; 4686 struct ice_rx_queue *rxq = pf->fdir.rxq; 4687 volatile struct ice_fltr_desc *fdirdp; 4688 volatile struct ice_tx_desc *txdp; 4689 uint32_t td_cmd; 4690 uint16_t i; 4691 4692 fdirdp = (volatile struct ice_fltr_desc *) 4693 (&txq->ice_tx_ring[txq->tx_tail]); 4694 fdirdp->qidx_compq_space_stat = fdir_desc->qidx_compq_space_stat; 4695 fdirdp->dtype_cmd_vsi_fdid = fdir_desc->dtype_cmd_vsi_fdid; 4696 4697 txdp = &txq->ice_tx_ring[txq->tx_tail + 1]; 4698 txdp->buf_addr = rte_cpu_to_le_64(pf->fdir.dma_addr); 4699 td_cmd = ICE_TX_DESC_CMD_EOP | 4700 ICE_TX_DESC_CMD_RS | 4701 ICE_TX_DESC_CMD_DUMMY; 4702 4703 txdp->cmd_type_offset_bsz = 4704 ice_build_ctob(td_cmd, 0, ICE_FDIR_PKT_LEN, 0); 4705 4706 txq->tx_tail += 2; 4707 if (txq->tx_tail >= txq->nb_tx_desc) 4708 txq->tx_tail = 0; 4709 /* Update the tx tail register */ 4710 ICE_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); 4711 for (i = 0; i < ICE_FDIR_MAX_WAIT_US; i++) { 4712 if ((txdp->cmd_type_offset_bsz & 4713 rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) == 4714 rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE)) 4715 break; 4716 rte_delay_us(1); 4717 } 4718 if (i >= ICE_FDIR_MAX_WAIT_US) { 4719 PMD_DRV_LOG(ERR, 4720 "Failed to program FDIR filter: time out to get DD on tx queue."); 4721 return -ETIMEDOUT; 4722 } 4723 4724 for (; i < ICE_FDIR_MAX_WAIT_US; i++) { 4725 int ret; 4726 4727 ret = ice_check_fdir_programming_status(rxq); 4728 if (ret == -EAGAIN) 4729 rte_delay_us(1); 4730 else 4731 return ret; 4732 } 4733 4734 PMD_DRV_LOG(ERR, 4735 "Failed to program FDIR filter: programming status reported."); 4736 return -ETIMEDOUT; 4737 4738 4739 } 4740