1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdint.h> 9 #include <rte_ethdev.h> 10 #include <ethdev_driver.h> 11 #include <rte_malloc.h> 12 #include <rte_net.h> 13 14 #include "ngbe_logs.h" 15 #include "base/ngbe.h" 16 #include "ngbe_ethdev.h" 17 #include "ngbe_rxtx.h" 18 19 #ifdef RTE_LIBRTE_IEEE1588 20 #define NGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST 21 #else 22 #define NGBE_TX_IEEE1588_TMST 0 23 #endif 24 25 /* Bit Mask to indicate what bits required for building Tx context */ 26 static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM | 27 RTE_MBUF_F_TX_OUTER_IPV6 | 28 RTE_MBUF_F_TX_OUTER_IPV4 | 29 RTE_MBUF_F_TX_IPV6 | 30 RTE_MBUF_F_TX_IPV4 | 31 RTE_MBUF_F_TX_VLAN | 32 RTE_MBUF_F_TX_L4_MASK | 33 RTE_MBUF_F_TX_TCP_SEG | 34 RTE_MBUF_F_TX_TUNNEL_MASK | 35 RTE_MBUF_F_TX_OUTER_IP_CKSUM | 36 NGBE_TX_IEEE1588_TMST); 37 38 #define NGBE_TX_OFFLOAD_NOTSUP_MASK \ 39 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ NGBE_TX_OFFLOAD_MASK) 40 41 /* 42 * Prefetch a cache line into all cache levels. 43 */ 44 #define rte_ngbe_prefetch(p) rte_prefetch0(p) 45 46 /********************************************************************* 47 * 48 * Tx functions 49 * 50 **********************************************************************/ 51 52 /* 53 * Check for descriptors with their DD bit set and free mbufs. 54 * Return the total number of buffers freed. 55 */ 56 static __rte_always_inline int 57 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq) 58 { 59 struct ngbe_tx_entry *txep; 60 uint32_t status; 61 int i, nb_free = 0; 62 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ]; 63 64 /* check DD bit on threshold descriptor */ 65 status = txq->tx_ring[txq->tx_next_dd].dw3; 66 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 67 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 68 ngbe_set32_masked(txq->tdc_reg_addr, 69 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 70 return 0; 71 } 72 73 /* 74 * first buffer to free from S/W ring is at index 75 * tx_next_dd - (tx_free_thresh-1) 76 */ 77 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 78 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 79 /* free buffers one at a time */ 80 m = rte_pktmbuf_prefree_seg(txep->mbuf); 81 txep->mbuf = NULL; 82 83 if (unlikely(m == NULL)) 84 continue; 85 86 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ || 87 (nb_free > 0 && m->pool != free[0]->pool)) { 88 rte_mempool_put_bulk(free[0]->pool, 89 (void **)free, nb_free); 90 nb_free = 0; 91 } 92 93 free[nb_free++] = m; 94 } 95 96 if (nb_free > 0) 97 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 98 99 /* buffers were freed, update counters */ 100 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 101 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 102 if (txq->tx_next_dd >= txq->nb_tx_desc) 103 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 104 105 return txq->tx_free_thresh; 106 } 107 108 /* Populate 4 descriptors with data from 4 mbufs */ 109 static inline void 110 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 111 { 112 uint64_t buf_dma_addr; 113 uint32_t pkt_len; 114 int i; 115 116 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 117 buf_dma_addr = rte_mbuf_data_iova(*pkts); 118 pkt_len = (*pkts)->data_len; 119 120 /* write data to descriptor */ 121 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 122 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 123 NGBE_TXD_DATLEN(pkt_len)); 124 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 125 126 rte_prefetch0(&(*pkts)->pool); 127 } 128 } 129 130 /* Populate 1 descriptor with data from 1 mbuf */ 131 static inline void 132 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 133 { 134 uint64_t buf_dma_addr; 135 uint32_t pkt_len; 136 137 buf_dma_addr = rte_mbuf_data_iova(*pkts); 138 pkt_len = (*pkts)->data_len; 139 140 /* write data to descriptor */ 141 txdp->qw0 = cpu_to_le64(buf_dma_addr); 142 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 143 NGBE_TXD_DATLEN(pkt_len)); 144 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 145 146 rte_prefetch0(&(*pkts)->pool); 147 } 148 149 /* 150 * Fill H/W descriptor ring with mbuf data. 151 * Copy mbuf pointers to the S/W ring. 152 */ 153 static inline void 154 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts, 155 uint16_t nb_pkts) 156 { 157 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 158 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 159 const int N_PER_LOOP = 4; 160 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 161 int mainpart, leftover; 162 int i, j; 163 164 /* 165 * Process most of the packets in chunks of N pkts. Any 166 * leftover packets will get processed one at a time. 167 */ 168 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 169 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 170 for (i = 0; i < mainpart; i += N_PER_LOOP) { 171 /* Copy N mbuf pointers to the S/W ring */ 172 for (j = 0; j < N_PER_LOOP; ++j) 173 (txep + i + j)->mbuf = *(pkts + i + j); 174 tx4(txdp + i, pkts + i); 175 } 176 177 if (unlikely(leftover > 0)) { 178 for (i = 0; i < leftover; ++i) { 179 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 180 tx1(txdp + mainpart + i, pkts + mainpart + i); 181 } 182 } 183 } 184 185 static inline uint16_t 186 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 187 uint16_t nb_pkts) 188 { 189 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 190 uint16_t n = 0; 191 192 /* 193 * Begin scanning the H/W ring for done descriptors when the 194 * number of available descriptors drops below tx_free_thresh. 195 * For each done descriptor, free the associated buffer. 196 */ 197 if (txq->nb_tx_free < txq->tx_free_thresh) 198 ngbe_tx_free_bufs(txq); 199 200 /* Only use descriptors that are available */ 201 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 202 if (unlikely(nb_pkts == 0)) 203 return 0; 204 205 /* Use exactly nb_pkts descriptors */ 206 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 207 208 /* 209 * At this point, we know there are enough descriptors in the 210 * ring to transmit all the packets. This assumes that each 211 * mbuf contains a single segment, and that no new offloads 212 * are expected, which would require a new context descriptor. 213 */ 214 215 /* 216 * See if we're going to wrap-around. If so, handle the top 217 * of the descriptor ring first, then do the bottom. If not, 218 * the processing looks just like the "bottom" part anyway... 219 */ 220 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 221 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 222 ngbe_tx_fill_hw_ring(txq, tx_pkts, n); 223 txq->tx_tail = 0; 224 } 225 226 /* Fill H/W descriptor ring with mbuf data */ 227 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 228 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 229 230 /* 231 * Check for wrap-around. This would only happen if we used 232 * up to the last descriptor in the ring, no more, no less. 233 */ 234 if (txq->tx_tail >= txq->nb_tx_desc) 235 txq->tx_tail = 0; 236 237 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 238 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 239 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 240 241 /* update tail pointer */ 242 rte_wmb(); 243 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 244 245 return nb_pkts; 246 } 247 248 uint16_t 249 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 250 uint16_t nb_pkts) 251 { 252 uint16_t nb_tx; 253 254 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 255 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST)) 256 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 257 258 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 259 nb_tx = 0; 260 while (nb_pkts != 0) { 261 uint16_t ret, n; 262 263 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST); 264 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 265 nb_tx = (uint16_t)(nb_tx + ret); 266 nb_pkts = (uint16_t)(nb_pkts - ret); 267 if (ret < n) 268 break; 269 } 270 271 return nb_tx; 272 } 273 274 static inline void 275 ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq, 276 volatile struct ngbe_tx_ctx_desc *ctx_txd, 277 uint64_t ol_flags, union ngbe_tx_offload tx_offload) 278 { 279 union ngbe_tx_offload tx_offload_mask; 280 uint32_t type_tucmd_mlhl; 281 uint32_t mss_l4len_idx; 282 uint32_t ctx_idx; 283 uint32_t vlan_macip_lens; 284 uint32_t tunnel_seed; 285 286 ctx_idx = txq->ctx_curr; 287 tx_offload_mask.data[0] = 0; 288 tx_offload_mask.data[1] = 0; 289 290 /* Specify which HW CTX to upload. */ 291 mss_l4len_idx = NGBE_TXD_IDX(ctx_idx); 292 type_tucmd_mlhl = NGBE_TXD_CTXT; 293 294 tx_offload_mask.ptid |= ~0; 295 type_tucmd_mlhl |= NGBE_TXD_PTID(tx_offload.ptid); 296 297 /* check if TCP segmentation required for this packet */ 298 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 299 tx_offload_mask.l2_len |= ~0; 300 tx_offload_mask.l3_len |= ~0; 301 tx_offload_mask.l4_len |= ~0; 302 tx_offload_mask.tso_segsz |= ~0; 303 mss_l4len_idx |= NGBE_TXD_MSS(tx_offload.tso_segsz); 304 mss_l4len_idx |= NGBE_TXD_L4LEN(tx_offload.l4_len); 305 } else { /* no TSO, check if hardware checksum is needed */ 306 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 307 tx_offload_mask.l2_len |= ~0; 308 tx_offload_mask.l3_len |= ~0; 309 } 310 311 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 312 case RTE_MBUF_F_TX_UDP_CKSUM: 313 mss_l4len_idx |= 314 NGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr)); 315 tx_offload_mask.l2_len |= ~0; 316 tx_offload_mask.l3_len |= ~0; 317 break; 318 case RTE_MBUF_F_TX_TCP_CKSUM: 319 mss_l4len_idx |= 320 NGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr)); 321 tx_offload_mask.l2_len |= ~0; 322 tx_offload_mask.l3_len |= ~0; 323 break; 324 case RTE_MBUF_F_TX_SCTP_CKSUM: 325 mss_l4len_idx |= 326 NGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr)); 327 tx_offload_mask.l2_len |= ~0; 328 tx_offload_mask.l3_len |= ~0; 329 break; 330 default: 331 break; 332 } 333 } 334 335 vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1); 336 337 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 338 tx_offload_mask.outer_tun_len |= ~0; 339 tx_offload_mask.outer_l2_len |= ~0; 340 tx_offload_mask.outer_l3_len |= ~0; 341 tx_offload_mask.l2_len |= ~0; 342 tunnel_seed = NGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1); 343 tunnel_seed |= NGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2); 344 345 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 346 case RTE_MBUF_F_TX_TUNNEL_IPIP: 347 /* for non UDP / GRE tunneling, set to 0b */ 348 break; 349 default: 350 PMD_TX_LOG(ERR, "Tunnel type not supported"); 351 return; 352 } 353 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.outer_l2_len); 354 } else { 355 tunnel_seed = 0; 356 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len); 357 } 358 359 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 360 tx_offload_mask.vlan_tci |= ~0; 361 vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci); 362 } 363 364 txq->ctx_cache[ctx_idx].flags = ol_flags; 365 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 366 tx_offload_mask.data[0] & tx_offload.data[0]; 367 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 368 tx_offload_mask.data[1] & tx_offload.data[1]; 369 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 370 371 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens); 372 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed); 373 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl); 374 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx); 375 } 376 377 /* 378 * Check which hardware context can be used. Use the existing match 379 * or create a new context descriptor. 380 */ 381 static inline uint32_t 382 what_ctx_update(struct ngbe_tx_queue *txq, uint64_t flags, 383 union ngbe_tx_offload tx_offload) 384 { 385 /* If match with the current used context */ 386 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 387 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 388 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 389 & tx_offload.data[0])) && 390 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 391 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 392 & tx_offload.data[1])))) 393 return txq->ctx_curr; 394 395 /* What if match with the next context */ 396 txq->ctx_curr ^= 1; 397 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 398 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 399 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 400 & tx_offload.data[0])) && 401 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 402 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 403 & tx_offload.data[1])))) 404 return txq->ctx_curr; 405 406 /* Mismatch, use the previous context */ 407 return NGBE_CTX_NUM; 408 } 409 410 static inline uint32_t 411 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 412 { 413 uint32_t tmp = 0; 414 415 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) { 416 tmp |= NGBE_TXD_CC; 417 tmp |= NGBE_TXD_L4CS; 418 } 419 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 420 tmp |= NGBE_TXD_CC; 421 tmp |= NGBE_TXD_IPCS; 422 } 423 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 424 tmp |= NGBE_TXD_CC; 425 tmp |= NGBE_TXD_EIPCS; 426 } 427 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 428 tmp |= NGBE_TXD_CC; 429 /* implies IPv4 cksum */ 430 if (ol_flags & RTE_MBUF_F_TX_IPV4) 431 tmp |= NGBE_TXD_IPCS; 432 tmp |= NGBE_TXD_L4CS; 433 } 434 if (ol_flags & RTE_MBUF_F_TX_VLAN) 435 tmp |= NGBE_TXD_CC; 436 437 return tmp; 438 } 439 440 static inline uint32_t 441 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 442 { 443 uint32_t cmdtype = 0; 444 445 if (ol_flags & RTE_MBUF_F_TX_VLAN) 446 cmdtype |= NGBE_TXD_VLE; 447 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 448 cmdtype |= NGBE_TXD_TSE; 449 return cmdtype; 450 } 451 452 static inline uint8_t 453 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) 454 { 455 bool tun; 456 457 if (ptype) 458 return ngbe_encode_ptype(ptype); 459 460 /* Only support flags in NGBE_TX_OFFLOAD_MASK */ 461 tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK); 462 463 /* L2 level */ 464 ptype = RTE_PTYPE_L2_ETHER; 465 if (oflags & RTE_MBUF_F_TX_VLAN) 466 ptype |= RTE_PTYPE_L2_ETHER_VLAN; 467 468 /* L3 level */ 469 if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM)) 470 ptype |= RTE_PTYPE_L3_IPV4; 471 else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6)) 472 ptype |= RTE_PTYPE_L3_IPV6; 473 474 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM)) 475 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4); 476 else if (oflags & (RTE_MBUF_F_TX_IPV6)) 477 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6); 478 479 /* L4 level */ 480 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) { 481 case RTE_MBUF_F_TX_TCP_CKSUM: 482 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 483 break; 484 case RTE_MBUF_F_TX_UDP_CKSUM: 485 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP); 486 break; 487 case RTE_MBUF_F_TX_SCTP_CKSUM: 488 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP); 489 break; 490 } 491 492 if (oflags & RTE_MBUF_F_TX_TCP_SEG) 493 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 494 495 /* Tunnel */ 496 switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) { 497 case RTE_MBUF_F_TX_TUNNEL_IPIP: 498 case RTE_MBUF_F_TX_TUNNEL_IP: 499 ptype |= RTE_PTYPE_L2_ETHER | 500 RTE_PTYPE_L3_IPV4 | 501 RTE_PTYPE_TUNNEL_IP; 502 break; 503 } 504 505 return ngbe_encode_ptype(ptype); 506 } 507 508 /* Reset transmit descriptors after they have been used */ 509 static inline int 510 ngbe_xmit_cleanup(struct ngbe_tx_queue *txq) 511 { 512 struct ngbe_tx_entry *sw_ring = txq->sw_ring; 513 volatile struct ngbe_tx_desc *txr = txq->tx_ring; 514 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 515 uint16_t nb_tx_desc = txq->nb_tx_desc; 516 uint16_t desc_to_clean_to; 517 uint16_t nb_tx_to_clean; 518 uint32_t status; 519 520 /* Determine the last descriptor needing to be cleaned */ 521 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh); 522 if (desc_to_clean_to >= nb_tx_desc) 523 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 524 525 /* Check to make sure the last descriptor to clean is done */ 526 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 527 status = txr[desc_to_clean_to].dw3; 528 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 529 PMD_TX_LOG(DEBUG, 530 "Tx descriptor %4u is not done" 531 "(port=%d queue=%d)", 532 desc_to_clean_to, 533 txq->port_id, txq->queue_id); 534 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 535 ngbe_set32_masked(txq->tdc_reg_addr, 536 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 537 /* Failed to clean any descriptors, better luck next time */ 538 return -(1); 539 } 540 541 /* Figure out how many descriptors will be cleaned */ 542 if (last_desc_cleaned > desc_to_clean_to) 543 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 544 desc_to_clean_to); 545 else 546 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 547 last_desc_cleaned); 548 549 PMD_TX_LOG(DEBUG, 550 "Cleaning %4u Tx descriptors: %4u to %4u (port=%d queue=%d)", 551 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 552 txq->port_id, txq->queue_id); 553 554 /* 555 * The last descriptor to clean is done, so that means all the 556 * descriptors from the last descriptor that was cleaned 557 * up to the last descriptor with the RS bit set 558 * are done. Only reset the threshold descriptor. 559 */ 560 txr[desc_to_clean_to].dw3 = 0; 561 562 /* Update the txq to reflect the last descriptor that was cleaned */ 563 txq->last_desc_cleaned = desc_to_clean_to; 564 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 565 566 /* No Error */ 567 return 0; 568 } 569 570 uint16_t 571 ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 572 uint16_t nb_pkts) 573 { 574 struct ngbe_tx_queue *txq; 575 struct ngbe_tx_entry *sw_ring; 576 struct ngbe_tx_entry *txe, *txn; 577 volatile struct ngbe_tx_desc *txr; 578 volatile struct ngbe_tx_desc *txd; 579 struct rte_mbuf *tx_pkt; 580 struct rte_mbuf *m_seg; 581 uint64_t buf_dma_addr; 582 uint32_t olinfo_status; 583 uint32_t cmd_type_len; 584 uint32_t pkt_len; 585 uint16_t slen; 586 uint64_t ol_flags; 587 uint16_t tx_id; 588 uint16_t tx_last; 589 uint16_t nb_tx; 590 uint16_t nb_used; 591 uint64_t tx_ol_req; 592 uint32_t ctx = 0; 593 uint32_t new_ctx; 594 union ngbe_tx_offload tx_offload; 595 596 tx_offload.data[0] = 0; 597 tx_offload.data[1] = 0; 598 txq = tx_queue; 599 sw_ring = txq->sw_ring; 600 txr = txq->tx_ring; 601 tx_id = txq->tx_tail; 602 txe = &sw_ring[tx_id]; 603 604 /* Determine if the descriptor ring needs to be cleaned. */ 605 if (txq->nb_tx_free < txq->tx_free_thresh) 606 ngbe_xmit_cleanup(txq); 607 608 rte_prefetch0(&txe->mbuf->pool); 609 610 /* Tx loop */ 611 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 612 new_ctx = 0; 613 tx_pkt = *tx_pkts++; 614 pkt_len = tx_pkt->pkt_len; 615 616 /* 617 * Determine how many (if any) context descriptors 618 * are needed for offload functionality. 619 */ 620 ol_flags = tx_pkt->ol_flags; 621 622 /* If hardware offload required */ 623 tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK; 624 if (tx_ol_req) { 625 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req, 626 tx_pkt->packet_type); 627 tx_offload.l2_len = tx_pkt->l2_len; 628 tx_offload.l3_len = tx_pkt->l3_len; 629 tx_offload.l4_len = tx_pkt->l4_len; 630 tx_offload.vlan_tci = tx_pkt->vlan_tci; 631 tx_offload.tso_segsz = tx_pkt->tso_segsz; 632 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 633 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 634 tx_offload.outer_tun_len = 0; 635 636 /* If new context need be built or reuse the exist ctx*/ 637 ctx = what_ctx_update(txq, tx_ol_req, tx_offload); 638 /* Only allocate context descriptor if required */ 639 new_ctx = (ctx == NGBE_CTX_NUM); 640 ctx = txq->ctx_curr; 641 } 642 643 /* 644 * Keep track of how many descriptors are used this loop 645 * This will always be the number of segments + the number of 646 * Context descriptors required to transmit the packet 647 */ 648 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 649 650 /* 651 * The number of descriptors that must be allocated for a 652 * packet is the number of segments of that packet, plus 1 653 * Context Descriptor for the hardware offload, if any. 654 * Determine the last Tx descriptor to allocate in the Tx ring 655 * for the packet, starting from the current position (tx_id) 656 * in the ring. 657 */ 658 tx_last = (uint16_t)(tx_id + nb_used - 1); 659 660 /* Circular ring */ 661 if (tx_last >= txq->nb_tx_desc) 662 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 663 664 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 665 " tx_first=%u tx_last=%u", 666 (uint16_t)txq->port_id, 667 (uint16_t)txq->queue_id, 668 (uint32_t)pkt_len, 669 (uint16_t)tx_id, 670 (uint16_t)tx_last); 671 672 /* 673 * Make sure there are enough Tx descriptors available to 674 * transmit the entire packet. 675 * nb_used better be less than or equal to txq->tx_free_thresh 676 */ 677 if (nb_used > txq->nb_tx_free) { 678 PMD_TX_LOG(DEBUG, 679 "Not enough free Tx descriptors " 680 "nb_used=%4u nb_free=%4u " 681 "(port=%d queue=%d)", 682 nb_used, txq->nb_tx_free, 683 txq->port_id, txq->queue_id); 684 685 if (ngbe_xmit_cleanup(txq) != 0) { 686 /* Could not clean any descriptors */ 687 if (nb_tx == 0) 688 return 0; 689 goto end_of_tx; 690 } 691 692 /* nb_used better be <= txq->tx_free_thresh */ 693 if (unlikely(nb_used > txq->tx_free_thresh)) { 694 PMD_TX_LOG(DEBUG, 695 "The number of descriptors needed to " 696 "transmit the packet exceeds the " 697 "RS bit threshold. This will impact " 698 "performance." 699 "nb_used=%4u nb_free=%4u " 700 "tx_free_thresh=%4u. " 701 "(port=%d queue=%d)", 702 nb_used, txq->nb_tx_free, 703 txq->tx_free_thresh, 704 txq->port_id, txq->queue_id); 705 /* 706 * Loop here until there are enough Tx 707 * descriptors or until the ring cannot be 708 * cleaned. 709 */ 710 while (nb_used > txq->nb_tx_free) { 711 if (ngbe_xmit_cleanup(txq) != 0) { 712 /* 713 * Could not clean any 714 * descriptors 715 */ 716 if (nb_tx == 0) 717 return 0; 718 goto end_of_tx; 719 } 720 } 721 } 722 } 723 724 /* 725 * By now there are enough free Tx descriptors to transmit 726 * the packet. 727 */ 728 729 /* 730 * Set common flags of all Tx Data Descriptors. 731 * 732 * The following bits must be set in the first Data Descriptor 733 * and are ignored in the other ones: 734 * - NGBE_TXD_FCS 735 * 736 * The following bits must only be set in the last Data 737 * Descriptor: 738 * - NGBE_TXD_EOP 739 */ 740 cmd_type_len = NGBE_TXD_FCS; 741 742 #ifdef RTE_LIBRTE_IEEE1588 743 if (ol_flags & PKT_TX_IEEE1588_TMST) 744 cmd_type_len |= NGBE_TXD_1588; 745 #endif 746 747 olinfo_status = 0; 748 if (tx_ol_req) { 749 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 750 /* when TSO is on, paylen in descriptor is the 751 * not the packet len but the tcp payload len 752 */ 753 pkt_len -= (tx_offload.l2_len + 754 tx_offload.l3_len + tx_offload.l4_len); 755 pkt_len -= 756 (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) 757 ? tx_offload.outer_l2_len + 758 tx_offload.outer_l3_len : 0; 759 } 760 761 /* 762 * Setup the Tx Context Descriptor if required 763 */ 764 if (new_ctx) { 765 volatile struct ngbe_tx_ctx_desc *ctx_txd; 766 767 ctx_txd = (volatile struct ngbe_tx_ctx_desc *) 768 &txr[tx_id]; 769 770 txn = &sw_ring[txe->next_id]; 771 rte_prefetch0(&txn->mbuf->pool); 772 773 if (txe->mbuf != NULL) { 774 rte_pktmbuf_free_seg(txe->mbuf); 775 txe->mbuf = NULL; 776 } 777 778 ngbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 779 tx_offload); 780 781 txe->last_id = tx_last; 782 tx_id = txe->next_id; 783 txe = txn; 784 } 785 786 /* 787 * Setup the Tx Data Descriptor, 788 * This path will go through 789 * whatever new/reuse the context descriptor 790 */ 791 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 792 olinfo_status |= 793 tx_desc_cksum_flags_to_olinfo(ol_flags); 794 olinfo_status |= NGBE_TXD_IDX(ctx); 795 } 796 797 olinfo_status |= NGBE_TXD_PAYLEN(pkt_len); 798 799 m_seg = tx_pkt; 800 do { 801 txd = &txr[tx_id]; 802 txn = &sw_ring[txe->next_id]; 803 rte_prefetch0(&txn->mbuf->pool); 804 805 if (txe->mbuf != NULL) 806 rte_pktmbuf_free_seg(txe->mbuf); 807 txe->mbuf = m_seg; 808 809 /* 810 * Set up Transmit Data Descriptor. 811 */ 812 slen = m_seg->data_len; 813 buf_dma_addr = rte_mbuf_data_iova(m_seg); 814 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr); 815 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen); 816 txd->dw3 = rte_cpu_to_le_32(olinfo_status); 817 txe->last_id = tx_last; 818 tx_id = txe->next_id; 819 txe = txn; 820 m_seg = m_seg->next; 821 } while (m_seg != NULL); 822 823 /* 824 * The last packet data descriptor needs End Of Packet (EOP) 825 */ 826 cmd_type_len |= NGBE_TXD_EOP; 827 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 828 829 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len); 830 } 831 832 end_of_tx: 833 834 rte_wmb(); 835 836 /* 837 * Set the Transmit Descriptor Tail (TDT) 838 */ 839 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 840 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 841 (uint16_t)tx_id, (uint16_t)nb_tx); 842 ngbe_set32_relaxed(txq->tdt_reg_addr, tx_id); 843 txq->tx_tail = tx_id; 844 845 return nb_tx; 846 } 847 848 /********************************************************************* 849 * 850 * Tx prep functions 851 * 852 **********************************************************************/ 853 uint16_t 854 ngbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 855 { 856 int i, ret; 857 uint64_t ol_flags; 858 struct rte_mbuf *m; 859 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 860 861 for (i = 0; i < nb_pkts; i++) { 862 m = tx_pkts[i]; 863 ol_flags = m->ol_flags; 864 865 /** 866 * Check if packet meets requirements for number of segments 867 * 868 * NOTE: for ngbe it's always (40 - WTHRESH) for both TSO and 869 * non-TSO 870 */ 871 872 if (m->nb_segs > NGBE_TX_MAX_SEG - txq->wthresh) { 873 rte_errno = -EINVAL; 874 return i; 875 } 876 877 if (ol_flags & NGBE_TX_OFFLOAD_NOTSUP_MASK) { 878 rte_errno = -ENOTSUP; 879 return i; 880 } 881 882 #ifdef RTE_ETHDEV_DEBUG_TX 883 ret = rte_validate_tx_offload(m); 884 if (ret != 0) { 885 rte_errno = ret; 886 return i; 887 } 888 #endif 889 ret = rte_net_intel_cksum_prepare(m); 890 if (ret != 0) { 891 rte_errno = ret; 892 return i; 893 } 894 } 895 896 return i; 897 } 898 899 /********************************************************************* 900 * 901 * Rx functions 902 * 903 **********************************************************************/ 904 static inline uint32_t 905 ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) 906 { 907 uint16_t ptid = NGBE_RXD_PTID(pkt_info); 908 909 ptid &= ptid_mask; 910 911 return ngbe_decode_ptype(ptid); 912 } 913 914 static inline uint64_t 915 ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) 916 { 917 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { 918 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 919 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 920 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 921 0, 0, 0, RTE_MBUF_F_RX_FDIR, 922 }; 923 #ifdef RTE_LIBRTE_IEEE1588 924 static uint64_t ip_pkt_etqf_map[8] = { 925 0, 0, 0, PKT_RX_IEEE1588_PTP, 926 0, 0, 0, 0, 927 }; 928 int etfid = ngbe_etflt_id(NGBE_RXD_PTID(pkt_info)); 929 if (likely(-1 != etfid)) 930 return ip_pkt_etqf_map[etfid] | 931 ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 932 else 933 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 934 #else 935 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 936 #endif 937 } 938 939 static inline uint64_t 940 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 941 { 942 uint64_t pkt_flags; 943 944 /* 945 * Check if VLAN present only. 946 * Do not check whether L3/L4 rx checksum done by NIC or not, 947 * That can be found from rte_eth_rxmode.offloads flag 948 */ 949 pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN && 950 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED) 951 ? vlan_flags : 0; 952 953 #ifdef RTE_LIBRTE_IEEE1588 954 if (rx_status & NGBE_RXD_STAT_1588) 955 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST; 956 #endif 957 return pkt_flags; 958 } 959 960 static inline uint64_t 961 rx_desc_error_to_pkt_flags(uint32_t rx_status) 962 { 963 uint64_t pkt_flags = 0; 964 965 /* checksum offload can't be disabled */ 966 if (rx_status & NGBE_RXD_STAT_IPCS) 967 pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS 968 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD); 969 970 if (rx_status & NGBE_RXD_STAT_L4CS) 971 pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS 972 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD); 973 974 if (rx_status & NGBE_RXD_STAT_EIPCS && 975 rx_status & NGBE_RXD_ERR_EIPCS) 976 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 977 978 return pkt_flags; 979 } 980 981 /* 982 * LOOK_AHEAD defines how many desc statuses to check beyond the 983 * current descriptor. 984 * It must be a pound define for optimal performance. 985 * Do not change the value of LOOK_AHEAD, as the ngbe_rx_scan_hw_ring 986 * function only works with LOOK_AHEAD=8. 987 */ 988 #define LOOK_AHEAD 8 989 #if (LOOK_AHEAD != 8) 990 #error "PMD NGBE: LOOK_AHEAD must be 8\n" 991 #endif 992 static inline int 993 ngbe_rx_scan_hw_ring(struct ngbe_rx_queue *rxq) 994 { 995 volatile struct ngbe_rx_desc *rxdp; 996 struct ngbe_rx_entry *rxep; 997 struct rte_mbuf *mb; 998 uint16_t pkt_len; 999 uint64_t pkt_flags; 1000 int nb_dd; 1001 uint32_t s[LOOK_AHEAD]; 1002 uint32_t pkt_info[LOOK_AHEAD]; 1003 int i, j, nb_rx = 0; 1004 uint32_t status; 1005 1006 /* get references to current descriptor and S/W ring entry */ 1007 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1008 rxep = &rxq->sw_ring[rxq->rx_tail]; 1009 1010 status = rxdp->qw1.lo.status; 1011 /* check to make sure there is at least 1 packet to receive */ 1012 if (!(status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1013 return 0; 1014 1015 /* 1016 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1017 * reference packets that are ready to be received. 1018 */ 1019 for (i = 0; i < RTE_PMD_NGBE_RX_MAX_BURST; 1020 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1021 /* Read desc statuses backwards to avoid race condition */ 1022 for (j = 0; j < LOOK_AHEAD; j++) 1023 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); 1024 1025 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 1026 1027 /* Compute how many status bits were set */ 1028 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1029 (s[nb_dd] & NGBE_RXD_STAT_DD); nb_dd++) 1030 ; 1031 1032 for (j = 0; j < nb_dd; j++) 1033 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0); 1034 1035 nb_rx += nb_dd; 1036 1037 /* Translate descriptor info to mbuf format */ 1038 for (j = 0; j < nb_dd; ++j) { 1039 mb = rxep[j].mbuf; 1040 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) - 1041 rxq->crc_len; 1042 mb->data_len = pkt_len; 1043 mb->pkt_len = pkt_len; 1044 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag); 1045 1046 /* convert descriptor fields to rte mbuf flags */ 1047 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1048 rxq->vlan_flags); 1049 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]); 1050 pkt_flags |= 1051 ngbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]); 1052 mb->ol_flags = pkt_flags; 1053 mb->packet_type = 1054 ngbe_rxd_pkt_info_to_pkt_type(pkt_info[j], 1055 NGBE_PTID_MASK); 1056 1057 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1058 mb->hash.rss = 1059 rte_le_to_cpu_32(rxdp[j].qw0.dw1); 1060 } 1061 1062 /* Move mbuf pointers from the S/W ring to the stage */ 1063 for (j = 0; j < LOOK_AHEAD; ++j) 1064 rxq->rx_stage[i + j] = rxep[j].mbuf; 1065 1066 /* stop if all requested packets could not be received */ 1067 if (nb_dd != LOOK_AHEAD) 1068 break; 1069 } 1070 1071 /* clear software ring entries so we can cleanup correctly */ 1072 for (i = 0; i < nb_rx; ++i) 1073 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1074 1075 return nb_rx; 1076 } 1077 1078 static inline int 1079 ngbe_rx_alloc_bufs(struct ngbe_rx_queue *rxq, bool reset_mbuf) 1080 { 1081 volatile struct ngbe_rx_desc *rxdp; 1082 struct ngbe_rx_entry *rxep; 1083 struct rte_mbuf *mb; 1084 uint16_t alloc_idx; 1085 __le64 dma_addr; 1086 int diag, i; 1087 1088 /* allocate buffers in bulk directly into the S/W ring */ 1089 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1090 rxep = &rxq->sw_ring[alloc_idx]; 1091 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1092 rxq->rx_free_thresh); 1093 if (unlikely(diag != 0)) 1094 return -ENOMEM; 1095 1096 rxdp = &rxq->rx_ring[alloc_idx]; 1097 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1098 /* populate the static rte mbuf fields */ 1099 mb = rxep[i].mbuf; 1100 if (reset_mbuf) 1101 mb->port = rxq->port_id; 1102 1103 rte_mbuf_refcnt_set(mb, 1); 1104 mb->data_off = RTE_PKTMBUF_HEADROOM; 1105 1106 /* populate the descriptors */ 1107 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1108 NGBE_RXD_HDRADDR(&rxdp[i], 0); 1109 NGBE_RXD_PKTADDR(&rxdp[i], dma_addr); 1110 } 1111 1112 /* update state of internal queue structure */ 1113 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1114 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1115 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1116 1117 /* no errors */ 1118 return 0; 1119 } 1120 1121 static inline uint16_t 1122 ngbe_rx_fill_from_stage(struct ngbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1123 uint16_t nb_pkts) 1124 { 1125 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1126 int i; 1127 1128 /* how many packets are ready to return? */ 1129 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1130 1131 /* copy mbuf pointers to the application's packet list */ 1132 for (i = 0; i < nb_pkts; ++i) 1133 rx_pkts[i] = stage[i]; 1134 1135 /* update internal queue state */ 1136 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1137 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1138 1139 return nb_pkts; 1140 } 1141 1142 static inline uint16_t 1143 ngbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1144 uint16_t nb_pkts) 1145 { 1146 struct ngbe_rx_queue *rxq = (struct ngbe_rx_queue *)rx_queue; 1147 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1148 uint16_t nb_rx = 0; 1149 1150 /* Any previously recv'd pkts will be returned from the Rx stage */ 1151 if (rxq->rx_nb_avail) 1152 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1153 1154 /* Scan the H/W ring for packets to receive */ 1155 nb_rx = (uint16_t)ngbe_rx_scan_hw_ring(rxq); 1156 1157 /* update internal queue state */ 1158 rxq->rx_next_avail = 0; 1159 rxq->rx_nb_avail = nb_rx; 1160 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1161 1162 /* if required, allocate new buffers to replenish descriptors */ 1163 if (rxq->rx_tail > rxq->rx_free_trigger) { 1164 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1165 1166 if (ngbe_rx_alloc_bufs(rxq, true) != 0) { 1167 int i, j; 1168 1169 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1170 "queue_id=%u", (uint16_t)rxq->port_id, 1171 (uint16_t)rxq->queue_id); 1172 1173 dev->data->rx_mbuf_alloc_failed += 1174 rxq->rx_free_thresh; 1175 1176 /* 1177 * Need to rewind any previous receives if we cannot 1178 * allocate new buffers to replenish the old ones. 1179 */ 1180 rxq->rx_nb_avail = 0; 1181 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1182 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1183 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1184 1185 return 0; 1186 } 1187 1188 /* update tail pointer */ 1189 rte_wmb(); 1190 ngbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger); 1191 } 1192 1193 if (rxq->rx_tail >= rxq->nb_rx_desc) 1194 rxq->rx_tail = 0; 1195 1196 /* received any packets this loop? */ 1197 if (rxq->rx_nb_avail) 1198 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1199 1200 return 0; 1201 } 1202 1203 /* split requests into chunks of size RTE_PMD_NGBE_RX_MAX_BURST */ 1204 uint16_t 1205 ngbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1206 uint16_t nb_pkts) 1207 { 1208 uint16_t nb_rx; 1209 1210 if (unlikely(nb_pkts == 0)) 1211 return 0; 1212 1213 if (likely(nb_pkts <= RTE_PMD_NGBE_RX_MAX_BURST)) 1214 return ngbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1215 1216 /* request is relatively large, chunk it up */ 1217 nb_rx = 0; 1218 while (nb_pkts) { 1219 uint16_t ret, n; 1220 1221 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_RX_MAX_BURST); 1222 ret = ngbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1223 nb_rx = (uint16_t)(nb_rx + ret); 1224 nb_pkts = (uint16_t)(nb_pkts - ret); 1225 if (ret < n) 1226 break; 1227 } 1228 1229 return nb_rx; 1230 } 1231 1232 uint16_t 1233 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1234 uint16_t nb_pkts) 1235 { 1236 struct ngbe_rx_queue *rxq; 1237 volatile struct ngbe_rx_desc *rx_ring; 1238 volatile struct ngbe_rx_desc *rxdp; 1239 struct ngbe_rx_entry *sw_ring; 1240 struct ngbe_rx_entry *rxe; 1241 struct rte_mbuf *rxm; 1242 struct rte_mbuf *nmb; 1243 struct ngbe_rx_desc rxd; 1244 uint64_t dma_addr; 1245 uint32_t staterr; 1246 uint32_t pkt_info; 1247 uint16_t pkt_len; 1248 uint16_t rx_id; 1249 uint16_t nb_rx; 1250 uint16_t nb_hold; 1251 uint64_t pkt_flags; 1252 1253 nb_rx = 0; 1254 nb_hold = 0; 1255 rxq = rx_queue; 1256 rx_id = rxq->rx_tail; 1257 rx_ring = rxq->rx_ring; 1258 sw_ring = rxq->sw_ring; 1259 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1260 while (nb_rx < nb_pkts) { 1261 /* 1262 * The order of operations here is important as the DD status 1263 * bit must not be read after any other descriptor fields. 1264 * rx_ring and rxdp are pointing to volatile data so the order 1265 * of accesses cannot be reordered by the compiler. If they were 1266 * not volatile, they could be reordered which could lead to 1267 * using invalid descriptor fields when read from rxd. 1268 */ 1269 rxdp = &rx_ring[rx_id]; 1270 staterr = rxdp->qw1.lo.status; 1271 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1272 break; 1273 rxd = *rxdp; 1274 1275 /* 1276 * End of packet. 1277 * 1278 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet 1279 * is likely to be invalid and to be dropped by the various 1280 * validation checks performed by the network stack. 1281 * 1282 * Allocate a new mbuf to replenish the RX ring descriptor. 1283 * If the allocation fails: 1284 * - arrange for that Rx descriptor to be the first one 1285 * being parsed the next time the receive function is 1286 * invoked [on the same queue]. 1287 * 1288 * - Stop parsing the Rx ring and return immediately. 1289 * 1290 * This policy do not drop the packet received in the Rx 1291 * descriptor for which the allocation of a new mbuf failed. 1292 * Thus, it allows that packet to be later retrieved if 1293 * mbuf have been freed in the mean time. 1294 * As a side effect, holding Rx descriptors instead of 1295 * systematically giving them back to the NIC may lead to 1296 * Rx ring exhaustion situations. 1297 * However, the NIC can gracefully prevent such situations 1298 * to happen by sending specific "back-pressure" flow control 1299 * frames to its peer(s). 1300 */ 1301 PMD_RX_LOG(DEBUG, 1302 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u", 1303 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1304 (uint16_t)rx_id, (uint32_t)staterr, 1305 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 1306 1307 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1308 if (nmb == NULL) { 1309 PMD_RX_LOG(DEBUG, 1310 "Rx mbuf alloc failed port_id=%u queue_id=%u", 1311 (uint16_t)rxq->port_id, 1312 (uint16_t)rxq->queue_id); 1313 dev->data->rx_mbuf_alloc_failed++; 1314 break; 1315 } 1316 1317 nb_hold++; 1318 rxe = &sw_ring[rx_id]; 1319 rx_id++; 1320 if (rx_id == rxq->nb_rx_desc) 1321 rx_id = 0; 1322 1323 /* Prefetch next mbuf while processing current one. */ 1324 rte_ngbe_prefetch(sw_ring[rx_id].mbuf); 1325 1326 /* 1327 * When next Rx descriptor is on a cache-line boundary, 1328 * prefetch the next 4 Rx descriptors and the next 8 pointers 1329 * to mbufs. 1330 */ 1331 if ((rx_id & 0x3) == 0) { 1332 rte_ngbe_prefetch(&rx_ring[rx_id]); 1333 rte_ngbe_prefetch(&sw_ring[rx_id]); 1334 } 1335 1336 rxm = rxe->mbuf; 1337 rxe->mbuf = nmb; 1338 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1339 NGBE_RXD_HDRADDR(rxdp, 0); 1340 NGBE_RXD_PKTADDR(rxdp, dma_addr); 1341 1342 /* 1343 * Initialize the returned mbuf. 1344 * 1) setup generic mbuf fields: 1345 * - number of segments, 1346 * - next segment, 1347 * - packet length, 1348 * - Rx port identifier. 1349 * 2) integrate hardware offload data, if any: 1350 * - RSS flag & hash, 1351 * - IP checksum flag, 1352 * - VLAN TCI, if any, 1353 * - error flags. 1354 */ 1355 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - 1356 rxq->crc_len); 1357 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1358 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1359 rxm->nb_segs = 1; 1360 rxm->next = NULL; 1361 rxm->pkt_len = pkt_len; 1362 rxm->data_len = pkt_len; 1363 rxm->port = rxq->port_id; 1364 1365 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); 1366 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ 1367 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); 1368 1369 pkt_flags = rx_desc_status_to_pkt_flags(staterr, 1370 rxq->vlan_flags); 1371 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1372 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1373 rxm->ol_flags = pkt_flags; 1374 rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1375 NGBE_PTID_MASK); 1376 1377 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1378 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); 1379 1380 /* 1381 * Store the mbuf address into the next entry of the array 1382 * of returned packets. 1383 */ 1384 rx_pkts[nb_rx++] = rxm; 1385 } 1386 rxq->rx_tail = rx_id; 1387 1388 /* 1389 * If the number of free Rx descriptors is greater than the Rx free 1390 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1391 * register. 1392 * Update the RDT with the value of the last processed Rx descriptor 1393 * minus 1, to guarantee that the RDT register is never equal to the 1394 * RDH register, which creates a "full" ring situation from the 1395 * hardware point of view... 1396 */ 1397 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1398 if (nb_hold > rxq->rx_free_thresh) { 1399 PMD_RX_LOG(DEBUG, 1400 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 1401 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1402 (uint16_t)rx_id, (uint16_t)nb_hold, 1403 (uint16_t)nb_rx); 1404 rx_id = (uint16_t)((rx_id == 0) ? 1405 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1406 ngbe_set32(rxq->rdt_reg_addr, rx_id); 1407 nb_hold = 0; 1408 } 1409 rxq->nb_rx_hold = nb_hold; 1410 return nb_rx; 1411 } 1412 1413 /** 1414 * ngbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1415 * 1416 * Fill the following info in the HEAD buffer of the Rx cluster: 1417 * - RX port identifier 1418 * - hardware offload data, if any: 1419 * - RSS flag & hash 1420 * - IP checksum flag 1421 * - VLAN TCI, if any 1422 * - error flags 1423 * @head HEAD of the packet cluster 1424 * @desc HW descriptor to get data from 1425 * @rxq Pointer to the Rx queue 1426 */ 1427 static inline void 1428 ngbe_fill_cluster_head_buf(struct rte_mbuf *head, struct ngbe_rx_desc *desc, 1429 struct ngbe_rx_queue *rxq, uint32_t staterr) 1430 { 1431 uint32_t pkt_info; 1432 uint64_t pkt_flags; 1433 1434 head->port = rxq->port_id; 1435 1436 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is 1437 * set in the pkt_flags field. 1438 */ 1439 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag); 1440 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0); 1441 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 1442 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1443 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1444 head->ol_flags = pkt_flags; 1445 head->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1446 NGBE_PTID_MASK); 1447 1448 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1449 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1); 1450 } 1451 1452 /** 1453 * ngbe_recv_pkts_sc - receive handler for scatter case. 1454 * 1455 * @rx_queue Rx queue handle 1456 * @rx_pkts table of received packets 1457 * @nb_pkts size of rx_pkts table 1458 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 1459 * 1460 * Returns the number of received packets/clusters (according to the "bulk 1461 * receive" interface). 1462 */ 1463 static inline uint16_t 1464 ngbe_recv_pkts_sc(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 1465 bool bulk_alloc) 1466 { 1467 struct ngbe_rx_queue *rxq = rx_queue; 1468 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1469 volatile struct ngbe_rx_desc *rx_ring = rxq->rx_ring; 1470 struct ngbe_rx_entry *sw_ring = rxq->sw_ring; 1471 struct ngbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 1472 uint16_t rx_id = rxq->rx_tail; 1473 uint16_t nb_rx = 0; 1474 uint16_t nb_hold = rxq->nb_rx_hold; 1475 uint16_t prev_id = rxq->rx_tail; 1476 1477 while (nb_rx < nb_pkts) { 1478 bool eop; 1479 struct ngbe_rx_entry *rxe; 1480 struct ngbe_scattered_rx_entry *sc_entry; 1481 struct ngbe_scattered_rx_entry *next_sc_entry = NULL; 1482 struct ngbe_rx_entry *next_rxe = NULL; 1483 struct rte_mbuf *first_seg; 1484 struct rte_mbuf *rxm; 1485 struct rte_mbuf *nmb = NULL; 1486 struct ngbe_rx_desc rxd; 1487 uint16_t data_len; 1488 uint16_t next_id; 1489 volatile struct ngbe_rx_desc *rxdp; 1490 uint32_t staterr; 1491 1492 next_desc: 1493 rxdp = &rx_ring[rx_id]; 1494 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); 1495 1496 if (!(staterr & NGBE_RXD_STAT_DD)) 1497 break; 1498 1499 rxd = *rxdp; 1500 1501 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1502 "staterr=0x%x data_len=%u", 1503 rxq->port_id, rxq->queue_id, rx_id, staterr, 1504 rte_le_to_cpu_16(rxd.qw1.hi.len)); 1505 1506 if (!bulk_alloc) { 1507 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1508 if (nmb == NULL) { 1509 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed " 1510 "port_id=%u queue_id=%u", 1511 rxq->port_id, rxq->queue_id); 1512 1513 dev->data->rx_mbuf_alloc_failed++; 1514 break; 1515 } 1516 } else if (nb_hold > rxq->rx_free_thresh) { 1517 uint16_t next_rdt = rxq->rx_free_trigger; 1518 1519 if (!ngbe_rx_alloc_bufs(rxq, false)) { 1520 rte_wmb(); 1521 ngbe_set32_relaxed(rxq->rdt_reg_addr, 1522 next_rdt); 1523 nb_hold -= rxq->rx_free_thresh; 1524 } else { 1525 PMD_RX_LOG(DEBUG, "Rx bulk alloc failed " 1526 "port_id=%u queue_id=%u", 1527 rxq->port_id, rxq->queue_id); 1528 1529 dev->data->rx_mbuf_alloc_failed++; 1530 break; 1531 } 1532 } 1533 1534 nb_hold++; 1535 rxe = &sw_ring[rx_id]; 1536 eop = staterr & NGBE_RXD_STAT_EOP; 1537 1538 next_id = rx_id + 1; 1539 if (next_id == rxq->nb_rx_desc) 1540 next_id = 0; 1541 1542 /* Prefetch next mbuf while processing current one. */ 1543 rte_ngbe_prefetch(sw_ring[next_id].mbuf); 1544 1545 /* 1546 * When next Rx descriptor is on a cache-line boundary, 1547 * prefetch the next 4 RX descriptors and the next 4 pointers 1548 * to mbufs. 1549 */ 1550 if ((next_id & 0x3) == 0) { 1551 rte_ngbe_prefetch(&rx_ring[next_id]); 1552 rte_ngbe_prefetch(&sw_ring[next_id]); 1553 } 1554 1555 rxm = rxe->mbuf; 1556 1557 if (!bulk_alloc) { 1558 __le64 dma = 1559 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1560 /* 1561 * Update Rx descriptor with the physical address of the 1562 * new data buffer of the new allocated mbuf. 1563 */ 1564 rxe->mbuf = nmb; 1565 1566 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1567 NGBE_RXD_HDRADDR(rxdp, 0); 1568 NGBE_RXD_PKTADDR(rxdp, dma); 1569 } else { 1570 rxe->mbuf = NULL; 1571 } 1572 1573 /* 1574 * Set data length & data buffer address of mbuf. 1575 */ 1576 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len); 1577 rxm->data_len = data_len; 1578 1579 if (!eop) { 1580 uint16_t nextp_id; 1581 1582 nextp_id = next_id; 1583 next_sc_entry = &sw_sc_ring[nextp_id]; 1584 next_rxe = &sw_ring[nextp_id]; 1585 rte_ngbe_prefetch(next_rxe); 1586 } 1587 1588 sc_entry = &sw_sc_ring[rx_id]; 1589 first_seg = sc_entry->fbuf; 1590 sc_entry->fbuf = NULL; 1591 1592 /* 1593 * If this is the first buffer of the received packet, 1594 * set the pointer to the first mbuf of the packet and 1595 * initialize its context. 1596 * Otherwise, update the total length and the number of segments 1597 * of the current scattered packet, and update the pointer to 1598 * the last mbuf of the current packet. 1599 */ 1600 if (first_seg == NULL) { 1601 first_seg = rxm; 1602 first_seg->pkt_len = data_len; 1603 first_seg->nb_segs = 1; 1604 } else { 1605 first_seg->pkt_len += data_len; 1606 first_seg->nb_segs++; 1607 } 1608 1609 prev_id = rx_id; 1610 rx_id = next_id; 1611 1612 /* 1613 * If this is not the last buffer of the received packet, update 1614 * the pointer to the first mbuf at the NEXTP entry in the 1615 * sw_sc_ring and continue to parse the Rx ring. 1616 */ 1617 if (!eop && next_rxe) { 1618 rxm->next = next_rxe->mbuf; 1619 next_sc_entry->fbuf = first_seg; 1620 goto next_desc; 1621 } 1622 1623 /* Initialize the first mbuf of the returned packet */ 1624 ngbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 1625 1626 /* Deal with the case, when HW CRC srip is disabled. */ 1627 first_seg->pkt_len -= rxq->crc_len; 1628 if (unlikely(rxm->data_len <= rxq->crc_len)) { 1629 struct rte_mbuf *lp; 1630 1631 for (lp = first_seg; lp->next != rxm; lp = lp->next) 1632 ; 1633 1634 first_seg->nb_segs--; 1635 lp->data_len -= rxq->crc_len - rxm->data_len; 1636 lp->next = NULL; 1637 rte_pktmbuf_free_seg(rxm); 1638 } else { 1639 rxm->data_len -= rxq->crc_len; 1640 } 1641 1642 /* Prefetch data of first segment, if configured to do so. */ 1643 rte_packet_prefetch((char *)first_seg->buf_addr + 1644 first_seg->data_off); 1645 1646 /* 1647 * Store the mbuf address into the next entry of the array 1648 * of returned packets. 1649 */ 1650 rx_pkts[nb_rx++] = first_seg; 1651 } 1652 1653 /* 1654 * Record index of the next Rx descriptor to probe. 1655 */ 1656 rxq->rx_tail = rx_id; 1657 1658 /* 1659 * If the number of free Rx descriptors is greater than the Rx free 1660 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1661 * register. 1662 * Update the RDT with the value of the last processed Rx descriptor 1663 * minus 1, to guarantee that the RDT register is never equal to the 1664 * RDH register, which creates a "full" ring situation from the 1665 * hardware point of view... 1666 */ 1667 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 1668 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1669 "nb_hold=%u nb_rx=%u", 1670 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 1671 1672 rte_wmb(); 1673 ngbe_set32_relaxed(rxq->rdt_reg_addr, prev_id); 1674 nb_hold = 0; 1675 } 1676 1677 rxq->nb_rx_hold = nb_hold; 1678 return nb_rx; 1679 } 1680 1681 uint16_t 1682 ngbe_recv_pkts_sc_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1683 uint16_t nb_pkts) 1684 { 1685 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, false); 1686 } 1687 1688 uint16_t 1689 ngbe_recv_pkts_sc_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1690 uint16_t nb_pkts) 1691 { 1692 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, true); 1693 } 1694 1695 /********************************************************************* 1696 * 1697 * Queue management functions 1698 * 1699 **********************************************************************/ 1700 1701 static void 1702 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq) 1703 { 1704 unsigned int i; 1705 1706 if (txq->sw_ring != NULL) { 1707 for (i = 0; i < txq->nb_tx_desc; i++) { 1708 if (txq->sw_ring[i].mbuf != NULL) { 1709 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 1710 txq->sw_ring[i].mbuf = NULL; 1711 } 1712 } 1713 } 1714 } 1715 1716 static void 1717 ngbe_tx_free_swring(struct ngbe_tx_queue *txq) 1718 { 1719 if (txq != NULL) 1720 rte_free(txq->sw_ring); 1721 } 1722 1723 static void 1724 ngbe_tx_queue_release(struct ngbe_tx_queue *txq) 1725 { 1726 if (txq != NULL) { 1727 if (txq->ops != NULL) { 1728 txq->ops->release_mbufs(txq); 1729 txq->ops->free_swring(txq); 1730 } 1731 rte_free(txq); 1732 } 1733 } 1734 1735 void 1736 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1737 { 1738 ngbe_tx_queue_release(dev->data->tx_queues[qid]); 1739 } 1740 1741 /* (Re)set dynamic ngbe_tx_queue fields to defaults */ 1742 static void 1743 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq) 1744 { 1745 static const struct ngbe_tx_desc zeroed_desc = {0}; 1746 struct ngbe_tx_entry *txe = txq->sw_ring; 1747 uint16_t prev, i; 1748 1749 /* Zero out HW ring memory */ 1750 for (i = 0; i < txq->nb_tx_desc; i++) 1751 txq->tx_ring[i] = zeroed_desc; 1752 1753 /* Initialize SW ring entries */ 1754 prev = (uint16_t)(txq->nb_tx_desc - 1); 1755 for (i = 0; i < txq->nb_tx_desc; i++) { 1756 /* the ring can also be modified by hardware */ 1757 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i]; 1758 1759 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD); 1760 txe[i].mbuf = NULL; 1761 txe[i].last_id = i; 1762 txe[prev].next_id = i; 1763 prev = i; 1764 } 1765 1766 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 1767 txq->tx_tail = 0; 1768 1769 /* 1770 * Always allow 1 descriptor to be un-allocated to avoid 1771 * a H/W race condition 1772 */ 1773 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 1774 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 1775 txq->ctx_curr = 0; 1776 memset((void *)&txq->ctx_cache, 0, 1777 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info)); 1778 } 1779 1780 static const struct ngbe_txq_ops def_txq_ops = { 1781 .release_mbufs = ngbe_tx_queue_release_mbufs, 1782 .free_swring = ngbe_tx_free_swring, 1783 .reset = ngbe_reset_tx_queue, 1784 }; 1785 1786 /* Takes an ethdev and a queue and sets up the tx function to be used based on 1787 * the queue parameters. Used in tx_queue_setup by primary process and then 1788 * in dev_init by secondary process when attaching to an existing ethdev. 1789 */ 1790 void 1791 ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq) 1792 { 1793 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 1794 if (txq->offloads == 0 && 1795 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) { 1796 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 1797 dev->tx_pkt_burst = ngbe_xmit_pkts_simple; 1798 dev->tx_pkt_prepare = NULL; 1799 } else { 1800 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 1801 PMD_INIT_LOG(DEBUG, 1802 " - offloads = 0x%" PRIx64, 1803 txq->offloads); 1804 PMD_INIT_LOG(DEBUG, 1805 " - tx_free_thresh = %lu [RTE_PMD_NGBE_TX_MAX_BURST=%lu]", 1806 (unsigned long)txq->tx_free_thresh, 1807 (unsigned long)RTE_PMD_NGBE_TX_MAX_BURST); 1808 dev->tx_pkt_burst = ngbe_xmit_pkts; 1809 dev->tx_pkt_prepare = ngbe_prep_pkts; 1810 } 1811 } 1812 1813 static const struct { 1814 eth_tx_burst_t pkt_burst; 1815 const char *info; 1816 } ngbe_tx_burst_infos[] = { 1817 { ngbe_xmit_pkts_simple, "Scalar Simple"}, 1818 { ngbe_xmit_pkts, "Scalar"}, 1819 }; 1820 1821 int 1822 ngbe_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 1823 struct rte_eth_burst_mode *mode) 1824 { 1825 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 1826 int ret = -EINVAL; 1827 unsigned int i; 1828 1829 for (i = 0; i < RTE_DIM(ngbe_tx_burst_infos); ++i) { 1830 if (pkt_burst == ngbe_tx_burst_infos[i].pkt_burst) { 1831 snprintf(mode->info, sizeof(mode->info), "%s", 1832 ngbe_tx_burst_infos[i].info); 1833 ret = 0; 1834 break; 1835 } 1836 } 1837 1838 return ret; 1839 } 1840 1841 uint64_t 1842 ngbe_get_tx_port_offloads(struct rte_eth_dev *dev) 1843 { 1844 uint64_t tx_offload_capa; 1845 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1846 1847 tx_offload_capa = 1848 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 1849 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 1850 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 1851 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 1852 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 1853 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | 1854 RTE_ETH_TX_OFFLOAD_TCP_TSO | 1855 RTE_ETH_TX_OFFLOAD_UDP_TSO | 1856 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO | 1857 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | 1858 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | 1859 RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 1860 1861 if (hw->is_pf) 1862 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT; 1863 1864 return tx_offload_capa; 1865 } 1866 1867 int 1868 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 1869 uint16_t queue_idx, 1870 uint16_t nb_desc, 1871 unsigned int socket_id, 1872 const struct rte_eth_txconf *tx_conf) 1873 { 1874 const struct rte_memzone *tz; 1875 struct ngbe_tx_queue *txq; 1876 struct ngbe_hw *hw; 1877 uint16_t tx_free_thresh; 1878 uint64_t offloads; 1879 1880 PMD_INIT_FUNC_TRACE(); 1881 hw = ngbe_dev_hw(dev); 1882 1883 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1884 1885 /* 1886 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh 1887 * descriptors are used or if the number of descriptors required 1888 * to transmit a packet is greater than the number of free Tx 1889 * descriptors. 1890 * One descriptor in the Tx ring is used as a sentinel to avoid a 1891 * H/W race condition, hence the maximum threshold constraints. 1892 * When set to zero use default values. 1893 */ 1894 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 1895 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 1896 if (tx_free_thresh >= (nb_desc - 3)) { 1897 PMD_INIT_LOG(ERR, 1898 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)", 1899 (unsigned int)tx_free_thresh, 1900 (int)dev->data->port_id, (int)queue_idx); 1901 return -(EINVAL); 1902 } 1903 1904 if (nb_desc % tx_free_thresh != 0) { 1905 PMD_INIT_LOG(ERR, 1906 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)", 1907 (unsigned int)tx_free_thresh, 1908 (int)dev->data->port_id, (int)queue_idx); 1909 return -(EINVAL); 1910 } 1911 1912 /* Free memory prior to re-allocation if needed... */ 1913 if (dev->data->tx_queues[queue_idx] != NULL) { 1914 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 1915 dev->data->tx_queues[queue_idx] = NULL; 1916 } 1917 1918 /* First allocate the Tx queue data structure */ 1919 txq = rte_zmalloc_socket("ethdev Tx queue", 1920 sizeof(struct ngbe_tx_queue), 1921 RTE_CACHE_LINE_SIZE, socket_id); 1922 if (txq == NULL) 1923 return -ENOMEM; 1924 1925 /* 1926 * Allocate Tx ring hardware descriptors. A memzone large enough to 1927 * handle the maximum ring size is allocated in order to allow for 1928 * resizing in later calls to the queue setup function. 1929 */ 1930 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 1931 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX, 1932 NGBE_ALIGN, socket_id); 1933 if (tz == NULL) { 1934 ngbe_tx_queue_release(txq); 1935 return -ENOMEM; 1936 } 1937 1938 txq->nb_tx_desc = nb_desc; 1939 txq->tx_free_thresh = tx_free_thresh; 1940 txq->pthresh = tx_conf->tx_thresh.pthresh; 1941 txq->hthresh = tx_conf->tx_thresh.hthresh; 1942 txq->wthresh = tx_conf->tx_thresh.wthresh; 1943 txq->queue_id = queue_idx; 1944 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 1945 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 1946 txq->port_id = dev->data->port_id; 1947 txq->offloads = offloads; 1948 txq->ops = &def_txq_ops; 1949 txq->tx_deferred_start = tx_conf->tx_deferred_start; 1950 1951 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx)); 1952 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx)); 1953 1954 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 1955 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz); 1956 1957 /* Allocate software ring */ 1958 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 1959 sizeof(struct ngbe_tx_entry) * nb_desc, 1960 RTE_CACHE_LINE_SIZE, socket_id); 1961 if (txq->sw_ring == NULL) { 1962 ngbe_tx_queue_release(txq); 1963 return -ENOMEM; 1964 } 1965 PMD_INIT_LOG(DEBUG, 1966 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 1967 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 1968 1969 /* set up scalar Tx function as appropriate */ 1970 ngbe_set_tx_function(dev, txq); 1971 1972 txq->ops->reset(txq); 1973 1974 dev->data->tx_queues[queue_idx] = txq; 1975 1976 return 0; 1977 } 1978 1979 /** 1980 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster 1981 * 1982 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 1983 * in the sw_sc_ring is not set to NULL but rather points to the next 1984 * mbuf of this RSC aggregation (that has not been completed yet and still 1985 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 1986 * will just free first "nb_segs" segments of the cluster explicitly by calling 1987 * an rte_pktmbuf_free_seg(). 1988 * 1989 * @m scattered cluster head 1990 */ 1991 static void 1992 ngbe_free_sc_cluster(struct rte_mbuf *m) 1993 { 1994 uint16_t i, nb_segs = m->nb_segs; 1995 struct rte_mbuf *next_seg; 1996 1997 for (i = 0; i < nb_segs; i++) { 1998 next_seg = m->next; 1999 rte_pktmbuf_free_seg(m); 2000 m = next_seg; 2001 } 2002 } 2003 2004 static void 2005 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq) 2006 { 2007 unsigned int i; 2008 2009 if (rxq->sw_ring != NULL) { 2010 for (i = 0; i < rxq->nb_rx_desc; i++) { 2011 if (rxq->sw_ring[i].mbuf != NULL) { 2012 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2013 rxq->sw_ring[i].mbuf = NULL; 2014 } 2015 } 2016 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2017 struct rte_mbuf *mb; 2018 2019 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2020 rte_pktmbuf_free_seg(mb); 2021 } 2022 rxq->rx_nb_avail = 0; 2023 } 2024 2025 if (rxq->sw_sc_ring != NULL) 2026 for (i = 0; i < rxq->nb_rx_desc; i++) 2027 if (rxq->sw_sc_ring[i].fbuf != NULL) { 2028 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2029 rxq->sw_sc_ring[i].fbuf = NULL; 2030 } 2031 } 2032 2033 static void 2034 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq) 2035 { 2036 if (rxq != NULL) { 2037 ngbe_rx_queue_release_mbufs(rxq); 2038 rte_free(rxq->sw_ring); 2039 rte_free(rxq->sw_sc_ring); 2040 rte_free(rxq); 2041 } 2042 } 2043 2044 void 2045 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2046 { 2047 ngbe_rx_queue_release(dev->data->rx_queues[qid]); 2048 } 2049 2050 /* 2051 * Check if Rx Burst Bulk Alloc function can be used. 2052 * Return 2053 * 0: the preconditions are satisfied and the bulk allocation function 2054 * can be used. 2055 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2056 * function must be used. 2057 */ 2058 static inline int 2059 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq) 2060 { 2061 int ret = 0; 2062 2063 /* 2064 * Make sure the following pre-conditions are satisfied: 2065 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST 2066 * rxq->rx_free_thresh < rxq->nb_rx_desc 2067 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2068 * Scattered packets are not supported. This should be checked 2069 * outside of this function. 2070 */ 2071 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) { 2072 PMD_INIT_LOG(DEBUG, 2073 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d", 2074 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST); 2075 ret = -EINVAL; 2076 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) { 2077 PMD_INIT_LOG(DEBUG, 2078 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d", 2079 rxq->rx_free_thresh, rxq->nb_rx_desc); 2080 ret = -EINVAL; 2081 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) { 2082 PMD_INIT_LOG(DEBUG, 2083 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d", 2084 rxq->nb_rx_desc, rxq->rx_free_thresh); 2085 ret = -EINVAL; 2086 } 2087 2088 return ret; 2089 } 2090 2091 /* Reset dynamic ngbe_rx_queue fields back to defaults */ 2092 static void 2093 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq) 2094 { 2095 static const struct ngbe_rx_desc zeroed_desc = { 2096 {{0}, {0} }, {{0}, {0} } }; 2097 unsigned int i; 2098 uint16_t len = rxq->nb_rx_desc; 2099 2100 /* 2101 * By default, the Rx queue setup function allocates enough memory for 2102 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 2103 * extra memory at the end of the descriptor ring to be zero'd out. 2104 */ 2105 if (adapter->rx_bulk_alloc_allowed) 2106 /* zero out extra memory */ 2107 len += RTE_PMD_NGBE_RX_MAX_BURST; 2108 2109 /* 2110 * Zero out HW ring memory. Zero out extra memory at the end of 2111 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2112 * reads extra memory as zeros. 2113 */ 2114 for (i = 0; i < len; i++) 2115 rxq->rx_ring[i] = zeroed_desc; 2116 2117 /* 2118 * initialize extra software ring entries. Space for these extra 2119 * entries is always allocated 2120 */ 2121 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2122 for (i = rxq->nb_rx_desc; i < len; ++i) 2123 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2124 2125 rxq->rx_nb_avail = 0; 2126 rxq->rx_next_avail = 0; 2127 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2128 rxq->rx_tail = 0; 2129 rxq->nb_rx_hold = 0; 2130 rxq->pkt_first_seg = NULL; 2131 rxq->pkt_last_seg = NULL; 2132 } 2133 2134 uint64_t 2135 ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused) 2136 { 2137 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2138 } 2139 2140 uint64_t 2141 ngbe_get_rx_port_offloads(struct rte_eth_dev *dev) 2142 { 2143 uint64_t offloads; 2144 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2145 2146 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 2147 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 2148 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 2149 RTE_ETH_RX_OFFLOAD_KEEP_CRC | 2150 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 2151 RTE_ETH_RX_OFFLOAD_SCATTER; 2152 2153 if (hw->is_pf) 2154 offloads |= (RTE_ETH_RX_OFFLOAD_QINQ_STRIP | 2155 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND); 2156 2157 return offloads; 2158 } 2159 2160 int 2161 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 2162 uint16_t queue_idx, 2163 uint16_t nb_desc, 2164 unsigned int socket_id, 2165 const struct rte_eth_rxconf *rx_conf, 2166 struct rte_mempool *mp) 2167 { 2168 const struct rte_memzone *rz; 2169 struct ngbe_rx_queue *rxq; 2170 struct ngbe_hw *hw; 2171 uint16_t len; 2172 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2173 uint64_t offloads; 2174 2175 PMD_INIT_FUNC_TRACE(); 2176 hw = ngbe_dev_hw(dev); 2177 2178 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2179 2180 /* Free memory prior to re-allocation if needed... */ 2181 if (dev->data->rx_queues[queue_idx] != NULL) { 2182 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 2183 dev->data->rx_queues[queue_idx] = NULL; 2184 } 2185 2186 /* First allocate the Rx queue data structure */ 2187 rxq = rte_zmalloc_socket("ethdev RX queue", 2188 sizeof(struct ngbe_rx_queue), 2189 RTE_CACHE_LINE_SIZE, socket_id); 2190 if (rxq == NULL) 2191 return -ENOMEM; 2192 rxq->mb_pool = mp; 2193 rxq->nb_rx_desc = nb_desc; 2194 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2195 rxq->queue_id = queue_idx; 2196 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2197 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2198 rxq->port_id = dev->data->port_id; 2199 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2200 rxq->crc_len = RTE_ETHER_CRC_LEN; 2201 else 2202 rxq->crc_len = 0; 2203 rxq->drop_en = rx_conf->rx_drop_en; 2204 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2205 rxq->offloads = offloads; 2206 2207 /* 2208 * Allocate Rx ring hardware descriptors. A memzone large enough to 2209 * handle the maximum ring size is allocated in order to allow for 2210 * resizing in later calls to the queue setup function. 2211 */ 2212 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2213 RX_RING_SZ, NGBE_ALIGN, socket_id); 2214 if (rz == NULL) { 2215 ngbe_rx_queue_release(rxq); 2216 return -ENOMEM; 2217 } 2218 2219 /* 2220 * Zero init all the descriptors in the ring. 2221 */ 2222 memset(rz->addr, 0, RX_RING_SZ); 2223 2224 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx)); 2225 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx)); 2226 2227 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 2228 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz); 2229 2230 /* 2231 * Certain constraints must be met in order to use the bulk buffer 2232 * allocation Rx burst function. If any of Rx queues doesn't meet them 2233 * the feature should be disabled for the whole port. 2234 */ 2235 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 2236 PMD_INIT_LOG(DEBUG, 2237 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]", 2238 rxq->queue_id, rxq->port_id); 2239 adapter->rx_bulk_alloc_allowed = false; 2240 } 2241 2242 /* 2243 * Allocate software ring. Allow for space at the end of the 2244 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 2245 * function does not access an invalid memory region. 2246 */ 2247 len = nb_desc; 2248 if (adapter->rx_bulk_alloc_allowed) 2249 len += RTE_PMD_NGBE_RX_MAX_BURST; 2250 2251 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 2252 sizeof(struct ngbe_rx_entry) * len, 2253 RTE_CACHE_LINE_SIZE, socket_id); 2254 if (rxq->sw_ring == NULL) { 2255 ngbe_rx_queue_release(rxq); 2256 return -ENOMEM; 2257 } 2258 2259 /* 2260 * Always allocate even if it's not going to be needed in order to 2261 * simplify the code. 2262 * 2263 * This ring is used in Scattered Rx cases and Scattered Rx may 2264 * be requested in ngbe_dev_rx_init(), which is called later from 2265 * dev_start() flow. 2266 */ 2267 rxq->sw_sc_ring = 2268 rte_zmalloc_socket("rxq->sw_sc_ring", 2269 sizeof(struct ngbe_scattered_rx_entry) * len, 2270 RTE_CACHE_LINE_SIZE, socket_id); 2271 if (rxq->sw_sc_ring == NULL) { 2272 ngbe_rx_queue_release(rxq); 2273 return -ENOMEM; 2274 } 2275 2276 PMD_INIT_LOG(DEBUG, 2277 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2278 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 2279 rxq->rx_ring_phys_addr); 2280 2281 dev->data->rx_queues[queue_idx] = rxq; 2282 2283 ngbe_reset_rx_queue(adapter, rxq); 2284 2285 return 0; 2286 } 2287 2288 void 2289 ngbe_dev_clear_queues(struct rte_eth_dev *dev) 2290 { 2291 unsigned int i; 2292 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2293 2294 PMD_INIT_FUNC_TRACE(); 2295 2296 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2297 struct ngbe_tx_queue *txq = dev->data->tx_queues[i]; 2298 2299 if (txq != NULL) { 2300 txq->ops->release_mbufs(txq); 2301 txq->ops->reset(txq); 2302 } 2303 } 2304 2305 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2306 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i]; 2307 2308 if (rxq != NULL) { 2309 ngbe_rx_queue_release_mbufs(rxq); 2310 ngbe_reset_rx_queue(adapter, rxq); 2311 } 2312 } 2313 } 2314 2315 void 2316 ngbe_dev_free_queues(struct rte_eth_dev *dev) 2317 { 2318 unsigned int i; 2319 2320 PMD_INIT_FUNC_TRACE(); 2321 2322 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2323 ngbe_dev_rx_queue_release(dev, i); 2324 dev->data->rx_queues[i] = NULL; 2325 } 2326 dev->data->nb_rx_queues = 0; 2327 2328 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2329 ngbe_dev_tx_queue_release(dev, i); 2330 dev->data->tx_queues[i] = NULL; 2331 } 2332 dev->data->nb_tx_queues = 0; 2333 } 2334 2335 /** 2336 * Receive Side Scaling (RSS) 2337 * 2338 * Principles: 2339 * The source and destination IP addresses of the IP header and the source 2340 * and destination ports of TCP/UDP headers, if any, of received packets are 2341 * hashed against a configurable random key to compute a 32-bit RSS hash result. 2342 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 2343 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 2344 * RSS output index which is used as the Rx queue index where to store the 2345 * received packets. 2346 * The following output is supplied in the Rx write-back descriptor: 2347 * - 32-bit result of the Microsoft RSS hash function, 2348 * - 4-bit RSS type field. 2349 */ 2350 2351 /* 2352 * Used as the default key. 2353 */ 2354 static uint8_t rss_intel_key[40] = { 2355 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 2356 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 2357 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 2358 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 2359 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 2360 }; 2361 2362 static void 2363 ngbe_rss_disable(struct rte_eth_dev *dev) 2364 { 2365 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2366 2367 wr32m(hw, NGBE_RACTL, NGBE_RACTL_RSSENA, 0); 2368 } 2369 2370 int 2371 ngbe_dev_rss_hash_update(struct rte_eth_dev *dev, 2372 struct rte_eth_rss_conf *rss_conf) 2373 { 2374 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2375 uint8_t *hash_key; 2376 uint32_t mrqc; 2377 uint32_t rss_key; 2378 uint64_t rss_hf; 2379 uint16_t i; 2380 2381 if (!hw->is_pf) { 2382 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 2383 "NIC."); 2384 return -ENOTSUP; 2385 } 2386 2387 hash_key = rss_conf->rss_key; 2388 if (hash_key) { 2389 /* Fill in RSS hash key */ 2390 for (i = 0; i < 10; i++) { 2391 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); 2392 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); 2393 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); 2394 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); 2395 wr32a(hw, NGBE_REG_RSSKEY, i, rss_key); 2396 } 2397 } 2398 2399 /* Set configured hashing protocols */ 2400 rss_hf = rss_conf->rss_hf & NGBE_RSS_OFFLOAD_ALL; 2401 2402 mrqc = rd32(hw, NGBE_RACTL); 2403 mrqc &= ~NGBE_RACTL_RSSMASK; 2404 if (rss_hf & RTE_ETH_RSS_IPV4) 2405 mrqc |= NGBE_RACTL_RSSIPV4; 2406 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 2407 mrqc |= NGBE_RACTL_RSSIPV4TCP; 2408 if (rss_hf & RTE_ETH_RSS_IPV6 || 2409 rss_hf & RTE_ETH_RSS_IPV6_EX) 2410 mrqc |= NGBE_RACTL_RSSIPV6; 2411 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP || 2412 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 2413 mrqc |= NGBE_RACTL_RSSIPV6TCP; 2414 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 2415 mrqc |= NGBE_RACTL_RSSIPV4UDP; 2416 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP || 2417 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 2418 mrqc |= NGBE_RACTL_RSSIPV6UDP; 2419 2420 if (rss_hf) 2421 mrqc |= NGBE_RACTL_RSSENA; 2422 else 2423 mrqc &= ~NGBE_RACTL_RSSENA; 2424 2425 wr32(hw, NGBE_RACTL, mrqc); 2426 2427 return 0; 2428 } 2429 2430 int 2431 ngbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2432 struct rte_eth_rss_conf *rss_conf) 2433 { 2434 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2435 uint8_t *hash_key; 2436 uint32_t mrqc; 2437 uint32_t rss_key; 2438 uint64_t rss_hf; 2439 uint16_t i; 2440 2441 hash_key = rss_conf->rss_key; 2442 if (hash_key) { 2443 /* Return RSS hash key */ 2444 for (i = 0; i < 10; i++) { 2445 rss_key = rd32a(hw, NGBE_REG_RSSKEY, i); 2446 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); 2447 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); 2448 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); 2449 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); 2450 } 2451 } 2452 2453 rss_hf = 0; 2454 2455 mrqc = rd32(hw, NGBE_RACTL); 2456 if (mrqc & NGBE_RACTL_RSSIPV4) 2457 rss_hf |= RTE_ETH_RSS_IPV4; 2458 if (mrqc & NGBE_RACTL_RSSIPV4TCP) 2459 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 2460 if (mrqc & NGBE_RACTL_RSSIPV6) 2461 rss_hf |= RTE_ETH_RSS_IPV6 | 2462 RTE_ETH_RSS_IPV6_EX; 2463 if (mrqc & NGBE_RACTL_RSSIPV6TCP) 2464 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | 2465 RTE_ETH_RSS_IPV6_TCP_EX; 2466 if (mrqc & NGBE_RACTL_RSSIPV4UDP) 2467 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 2468 if (mrqc & NGBE_RACTL_RSSIPV6UDP) 2469 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | 2470 RTE_ETH_RSS_IPV6_UDP_EX; 2471 if (!(mrqc & NGBE_RACTL_RSSENA)) 2472 rss_hf = 0; 2473 2474 rss_hf &= NGBE_RSS_OFFLOAD_ALL; 2475 2476 rss_conf->rss_hf = rss_hf; 2477 return 0; 2478 } 2479 2480 static void 2481 ngbe_rss_configure(struct rte_eth_dev *dev) 2482 { 2483 struct rte_eth_rss_conf rss_conf; 2484 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2485 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2486 uint32_t reta; 2487 uint16_t i; 2488 uint16_t j; 2489 2490 PMD_INIT_FUNC_TRACE(); 2491 2492 /* 2493 * Fill in redirection table 2494 * The byte-swap is needed because NIC registers are in 2495 * little-endian order. 2496 */ 2497 if (adapter->rss_reta_updated == 0) { 2498 reta = 0; 2499 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { 2500 if (j == dev->data->nb_rx_queues) 2501 j = 0; 2502 reta = (reta >> 8) | LS32(j, 24, 0xFF); 2503 if ((i & 3) == 3) 2504 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta); 2505 } 2506 } 2507 /* 2508 * Configure the RSS key and the RSS protocols used to compute 2509 * the RSS hash of input packets. 2510 */ 2511 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 2512 if (rss_conf.rss_key == NULL) 2513 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 2514 ngbe_dev_rss_hash_update(dev, &rss_conf); 2515 } 2516 2517 void ngbe_configure_port(struct rte_eth_dev *dev) 2518 { 2519 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2520 int i = 0; 2521 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, 2522 0x9100, 0x9200, 2523 0x0000, 0x0000, 2524 0x0000, 0x0000}; 2525 2526 PMD_INIT_FUNC_TRACE(); 2527 2528 /* default outer vlan tpid */ 2529 wr32(hw, NGBE_EXTAG, 2530 NGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | 2531 NGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); 2532 2533 /* default inner vlan tpid */ 2534 wr32m(hw, NGBE_VLANCTL, 2535 NGBE_VLANCTL_TPID_MASK, 2536 NGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); 2537 wr32m(hw, NGBE_DMATXCTRL, 2538 NGBE_DMATXCTRL_TPID_MASK, 2539 NGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); 2540 2541 /* default vlan tpid filters */ 2542 for (i = 0; i < 8; i++) { 2543 wr32m(hw, NGBE_TAGTPID(i / 2), 2544 (i % 2 ? NGBE_TAGTPID_MSB_MASK 2545 : NGBE_TAGTPID_LSB_MASK), 2546 (i % 2 ? NGBE_TAGTPID_MSB(tpids[i]) 2547 : NGBE_TAGTPID_LSB(tpids[i]))); 2548 } 2549 } 2550 2551 static int 2552 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq) 2553 { 2554 struct ngbe_rx_entry *rxe = rxq->sw_ring; 2555 uint64_t dma_addr; 2556 unsigned int i; 2557 2558 /* Initialize software ring entries */ 2559 for (i = 0; i < rxq->nb_rx_desc; i++) { 2560 /* the ring can also be modified by hardware */ 2561 volatile struct ngbe_rx_desc *rxd; 2562 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 2563 2564 if (mbuf == NULL) { 2565 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u", 2566 (unsigned int)rxq->queue_id, 2567 (unsigned int)rxq->port_id); 2568 return -ENOMEM; 2569 } 2570 2571 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2572 mbuf->port = rxq->port_id; 2573 2574 dma_addr = 2575 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 2576 rxd = &rxq->rx_ring[i]; 2577 NGBE_RXD_HDRADDR(rxd, 0); 2578 NGBE_RXD_PKTADDR(rxd, dma_addr); 2579 rxe[i].mbuf = mbuf; 2580 } 2581 2582 return 0; 2583 } 2584 2585 static int 2586 ngbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 2587 { 2588 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 2589 switch (dev->data->dev_conf.rxmode.mq_mode) { 2590 case RTE_ETH_MQ_RX_RSS: 2591 ngbe_rss_configure(dev); 2592 break; 2593 2594 case RTE_ETH_MQ_RX_NONE: 2595 default: 2596 /* if mq_mode is none, disable rss mode.*/ 2597 ngbe_rss_disable(dev); 2598 break; 2599 } 2600 } 2601 2602 return 0; 2603 } 2604 2605 void 2606 ngbe_set_rx_function(struct rte_eth_dev *dev) 2607 { 2608 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2609 2610 if (dev->data->scattered_rx) { 2611 /* 2612 * Set the scattered callback: there are bulk and 2613 * single allocation versions. 2614 */ 2615 if (adapter->rx_bulk_alloc_allowed) { 2616 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 2617 "allocation callback (port=%d).", 2618 dev->data->port_id); 2619 dev->rx_pkt_burst = ngbe_recv_pkts_sc_bulk_alloc; 2620 } else { 2621 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 2622 "single allocation) " 2623 "Scattered Rx callback " 2624 "(port=%d).", 2625 dev->data->port_id); 2626 2627 dev->rx_pkt_burst = ngbe_recv_pkts_sc_single_alloc; 2628 } 2629 /* 2630 * Below we set "simple" callbacks according to port/queues parameters. 2631 * If parameters allow we are going to choose between the following 2632 * callbacks: 2633 * - Bulk Allocation 2634 * - Single buffer allocation (the simplest one) 2635 */ 2636 } else if (adapter->rx_bulk_alloc_allowed) { 2637 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 2638 "satisfied. Rx Burst Bulk Alloc function " 2639 "will be used on port=%d.", 2640 dev->data->port_id); 2641 2642 dev->rx_pkt_burst = ngbe_recv_pkts_bulk_alloc; 2643 } else { 2644 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 2645 "satisfied, or Scattered Rx is requested " 2646 "(port=%d).", 2647 dev->data->port_id); 2648 2649 dev->rx_pkt_burst = ngbe_recv_pkts; 2650 } 2651 } 2652 2653 static const struct { 2654 eth_rx_burst_t pkt_burst; 2655 const char *info; 2656 } ngbe_rx_burst_infos[] = { 2657 { ngbe_recv_pkts_sc_single_alloc, "Scalar Scattered"}, 2658 { ngbe_recv_pkts_sc_bulk_alloc, "Scalar Scattered Bulk Alloc"}, 2659 { ngbe_recv_pkts_bulk_alloc, "Scalar Bulk Alloc"}, 2660 { ngbe_recv_pkts, "Scalar"}, 2661 }; 2662 2663 int 2664 ngbe_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 2665 struct rte_eth_burst_mode *mode) 2666 { 2667 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 2668 int ret = -EINVAL; 2669 unsigned int i; 2670 2671 for (i = 0; i < RTE_DIM(ngbe_rx_burst_infos); ++i) { 2672 if (pkt_burst == ngbe_rx_burst_infos[i].pkt_burst) { 2673 snprintf(mode->info, sizeof(mode->info), "%s", 2674 ngbe_rx_burst_infos[i].info); 2675 ret = 0; 2676 break; 2677 } 2678 } 2679 2680 return ret; 2681 } 2682 2683 /* 2684 * Initializes Receive Unit. 2685 */ 2686 int 2687 ngbe_dev_rx_init(struct rte_eth_dev *dev) 2688 { 2689 struct ngbe_hw *hw; 2690 struct ngbe_rx_queue *rxq; 2691 uint64_t bus_addr; 2692 uint32_t fctrl; 2693 uint32_t hlreg0; 2694 uint32_t srrctl; 2695 uint32_t rdrxctl; 2696 uint32_t rxcsum; 2697 uint16_t buf_size; 2698 uint16_t i; 2699 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 2700 2701 PMD_INIT_FUNC_TRACE(); 2702 hw = ngbe_dev_hw(dev); 2703 2704 /* 2705 * Make sure receives are disabled while setting 2706 * up the Rx context (registers, descriptor rings, etc.). 2707 */ 2708 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0); 2709 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0); 2710 2711 /* Enable receipt of broadcasted frames */ 2712 fctrl = rd32(hw, NGBE_PSRCTL); 2713 fctrl |= NGBE_PSRCTL_BCA; 2714 wr32(hw, NGBE_PSRCTL, fctrl); 2715 2716 /* 2717 * Configure CRC stripping, if any. 2718 */ 2719 hlreg0 = rd32(hw, NGBE_SECRXCTL); 2720 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2721 hlreg0 &= ~NGBE_SECRXCTL_CRCSTRIP; 2722 else 2723 hlreg0 |= NGBE_SECRXCTL_CRCSTRIP; 2724 hlreg0 &= ~NGBE_SECRXCTL_XDSA; 2725 wr32(hw, NGBE_SECRXCTL, hlreg0); 2726 2727 /* 2728 * Configure jumbo frame support, if any. 2729 */ 2730 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK, 2731 NGBE_FRMSZ_MAX(dev->data->mtu + NGBE_ETH_OVERHEAD)); 2732 2733 /* 2734 * If loopback mode is configured, set LPBK bit. 2735 */ 2736 hlreg0 = rd32(hw, NGBE_PSRCTL); 2737 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 2738 hlreg0 |= NGBE_PSRCTL_LBENA; 2739 else 2740 hlreg0 &= ~NGBE_PSRCTL_LBENA; 2741 2742 wr32(hw, NGBE_PSRCTL, hlreg0); 2743 2744 /* 2745 * Assume no header split and no VLAN strip support 2746 * on any Rx queue first . 2747 */ 2748 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2749 2750 /* Setup Rx queues */ 2751 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2752 rxq = dev->data->rx_queues[i]; 2753 2754 /* 2755 * Reset crc_len in case it was changed after queue setup by a 2756 * call to configure. 2757 */ 2758 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2759 rxq->crc_len = RTE_ETHER_CRC_LEN; 2760 else 2761 rxq->crc_len = 0; 2762 2763 /* Setup the Base and Length of the Rx Descriptor Rings */ 2764 bus_addr = rxq->rx_ring_phys_addr; 2765 wr32(hw, NGBE_RXBAL(rxq->reg_idx), 2766 (uint32_t)(bus_addr & BIT_MASK32)); 2767 wr32(hw, NGBE_RXBAH(rxq->reg_idx), 2768 (uint32_t)(bus_addr >> 32)); 2769 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 2770 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0); 2771 2772 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 2773 2774 /* Set if packets are dropped when no descriptors available */ 2775 if (rxq->drop_en) 2776 srrctl |= NGBE_RXCFG_DROP; 2777 2778 /* 2779 * Configure the Rx buffer size in the PKTLEN field of 2780 * the RXCFG register of the queue. 2781 * The value is in 1 KB resolution. Valid values can be from 2782 * 1 KB to 16 KB. 2783 */ 2784 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 2785 RTE_PKTMBUF_HEADROOM); 2786 buf_size = ROUND_DOWN(buf_size, 0x1 << 10); 2787 srrctl |= NGBE_RXCFG_PKTLEN(buf_size); 2788 2789 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl); 2790 2791 /* It adds dual VLAN length for supporting dual VLAN */ 2792 if (dev->data->mtu + NGBE_ETH_OVERHEAD + 2793 2 * NGBE_VLAN_TAG_SIZE > buf_size) 2794 dev->data->scattered_rx = 1; 2795 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 2796 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2797 } 2798 2799 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 2800 dev->data->scattered_rx = 1; 2801 2802 /* 2803 * Device configured with multiple RX queues. 2804 */ 2805 ngbe_dev_mq_rx_configure(dev); 2806 2807 /* 2808 * Setup the Checksum Register. 2809 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 2810 * Enable IP/L4 checksum computation by hardware if requested to do so. 2811 */ 2812 rxcsum = rd32(hw, NGBE_PSRCTL); 2813 rxcsum |= NGBE_PSRCTL_PCSD; 2814 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 2815 rxcsum |= NGBE_PSRCTL_L4CSUM; 2816 else 2817 rxcsum &= ~NGBE_PSRCTL_L4CSUM; 2818 2819 wr32(hw, NGBE_PSRCTL, rxcsum); 2820 2821 if (hw->is_pf) { 2822 rdrxctl = rd32(hw, NGBE_SECRXCTL); 2823 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2824 rdrxctl &= ~NGBE_SECRXCTL_CRCSTRIP; 2825 else 2826 rdrxctl |= NGBE_SECRXCTL_CRCSTRIP; 2827 wr32(hw, NGBE_SECRXCTL, rdrxctl); 2828 } 2829 2830 ngbe_set_rx_function(dev); 2831 2832 return 0; 2833 } 2834 2835 /* 2836 * Initializes Transmit Unit. 2837 */ 2838 void 2839 ngbe_dev_tx_init(struct rte_eth_dev *dev) 2840 { 2841 struct ngbe_hw *hw; 2842 struct ngbe_tx_queue *txq; 2843 uint64_t bus_addr; 2844 uint16_t i; 2845 2846 PMD_INIT_FUNC_TRACE(); 2847 hw = ngbe_dev_hw(dev); 2848 2849 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA); 2850 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0); 2851 2852 /* Setup the Base and Length of the Tx Descriptor Rings */ 2853 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2854 txq = dev->data->tx_queues[i]; 2855 2856 bus_addr = txq->tx_ring_phys_addr; 2857 wr32(hw, NGBE_TXBAL(txq->reg_idx), 2858 (uint32_t)(bus_addr & BIT_MASK32)); 2859 wr32(hw, NGBE_TXBAH(txq->reg_idx), 2860 (uint32_t)(bus_addr >> 32)); 2861 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK, 2862 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 2863 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 2864 wr32(hw, NGBE_TXRP(txq->reg_idx), 0); 2865 wr32(hw, NGBE_TXWP(txq->reg_idx), 0); 2866 } 2867 } 2868 2869 /* 2870 * Set up link loopback mode Tx->Rx. 2871 */ 2872 static inline void 2873 ngbe_setup_loopback_link(struct ngbe_hw *hw) 2874 { 2875 PMD_INIT_FUNC_TRACE(); 2876 2877 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_LB, NGBE_MACRXCFG_LB); 2878 2879 msec_delay(50); 2880 } 2881 2882 /* 2883 * Start Transmit and Receive Units. 2884 */ 2885 int 2886 ngbe_dev_rxtx_start(struct rte_eth_dev *dev) 2887 { 2888 struct ngbe_hw *hw; 2889 struct ngbe_tx_queue *txq; 2890 struct ngbe_rx_queue *rxq; 2891 uint32_t dmatxctl; 2892 uint32_t rxctrl; 2893 uint16_t i; 2894 int ret = 0; 2895 2896 PMD_INIT_FUNC_TRACE(); 2897 hw = ngbe_dev_hw(dev); 2898 2899 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2900 txq = dev->data->tx_queues[i]; 2901 /* Setup Transmit Threshold Registers */ 2902 wr32m(hw, NGBE_TXCFG(txq->reg_idx), 2903 NGBE_TXCFG_HTHRESH_MASK | 2904 NGBE_TXCFG_WTHRESH_MASK, 2905 NGBE_TXCFG_HTHRESH(txq->hthresh) | 2906 NGBE_TXCFG_WTHRESH(txq->wthresh)); 2907 } 2908 2909 dmatxctl = rd32(hw, NGBE_DMATXCTRL); 2910 dmatxctl |= NGBE_DMATXCTRL_ENA; 2911 wr32(hw, NGBE_DMATXCTRL, dmatxctl); 2912 2913 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2914 txq = dev->data->tx_queues[i]; 2915 if (txq->tx_deferred_start == 0) { 2916 ret = ngbe_dev_tx_queue_start(dev, i); 2917 if (ret < 0) 2918 return ret; 2919 } 2920 } 2921 2922 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2923 rxq = dev->data->rx_queues[i]; 2924 if (rxq->rx_deferred_start == 0) { 2925 ret = ngbe_dev_rx_queue_start(dev, i); 2926 if (ret < 0) 2927 return ret; 2928 } 2929 } 2930 2931 /* Enable Receive engine */ 2932 rxctrl = rd32(hw, NGBE_PBRXCTL); 2933 rxctrl |= NGBE_PBRXCTL_ENA; 2934 hw->mac.enable_rx_dma(hw, rxctrl); 2935 2936 /* If loopback mode is enabled, set up the link accordingly */ 2937 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 2938 ngbe_setup_loopback_link(hw); 2939 2940 return 0; 2941 } 2942 2943 void 2944 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 2945 { 2946 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 2947 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id)); 2948 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id)); 2949 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id)); 2950 } 2951 2952 void 2953 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 2954 { 2955 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 2956 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++)); 2957 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++)); 2958 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA); 2959 } 2960 2961 void 2962 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 2963 { 2964 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 2965 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id)); 2966 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id)); 2967 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id)); 2968 } 2969 2970 void 2971 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 2972 { 2973 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 2974 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++)); 2975 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++)); 2976 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA); 2977 } 2978 2979 /* 2980 * Start Receive Units for specified queue. 2981 */ 2982 int 2983 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 2984 { 2985 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2986 struct ngbe_rx_queue *rxq; 2987 uint32_t rxdctl; 2988 int poll_ms; 2989 2990 PMD_INIT_FUNC_TRACE(); 2991 2992 rxq = dev->data->rx_queues[rx_queue_id]; 2993 2994 /* Allocate buffers for descriptor rings */ 2995 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) { 2996 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 2997 rx_queue_id); 2998 return -1; 2999 } 3000 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3001 rxdctl |= NGBE_RXCFG_ENA; 3002 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl); 3003 3004 /* Wait until Rx Enable ready */ 3005 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3006 do { 3007 rte_delay_ms(1); 3008 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3009 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA)); 3010 if (poll_ms == 0) 3011 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 3012 rte_wmb(); 3013 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 3014 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 3015 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3016 3017 return 0; 3018 } 3019 3020 /* 3021 * Stop Receive Units for specified queue. 3022 */ 3023 int 3024 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3025 { 3026 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3027 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 3028 struct ngbe_rx_queue *rxq; 3029 uint32_t rxdctl; 3030 int poll_ms; 3031 3032 PMD_INIT_FUNC_TRACE(); 3033 3034 rxq = dev->data->rx_queues[rx_queue_id]; 3035 3036 ngbe_dev_save_rx_queue(hw, rxq->reg_idx); 3037 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0); 3038 3039 /* Wait until Rx Enable bit clear */ 3040 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3041 do { 3042 rte_delay_ms(1); 3043 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3044 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA)); 3045 if (poll_ms == 0) 3046 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 3047 3048 rte_delay_us(RTE_NGBE_WAIT_100_US); 3049 ngbe_dev_store_rx_queue(hw, rxq->reg_idx); 3050 3051 ngbe_rx_queue_release_mbufs(rxq); 3052 ngbe_reset_rx_queue(adapter, rxq); 3053 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3054 3055 return 0; 3056 } 3057 3058 /* 3059 * Start Transmit Units for specified queue. 3060 */ 3061 int 3062 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3063 { 3064 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3065 struct ngbe_tx_queue *txq; 3066 uint32_t txdctl; 3067 int poll_ms; 3068 3069 PMD_INIT_FUNC_TRACE(); 3070 3071 txq = dev->data->tx_queues[tx_queue_id]; 3072 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA); 3073 3074 /* Wait until Tx Enable ready */ 3075 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3076 do { 3077 rte_delay_ms(1); 3078 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3079 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA)); 3080 if (poll_ms == 0) 3081 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 3082 tx_queue_id); 3083 3084 rte_wmb(); 3085 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail); 3086 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3087 3088 return 0; 3089 } 3090 3091 /* 3092 * Stop Transmit Units for specified queue. 3093 */ 3094 int 3095 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3096 { 3097 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3098 struct ngbe_tx_queue *txq; 3099 uint32_t txdctl; 3100 uint32_t txtdh, txtdt; 3101 int poll_ms; 3102 3103 PMD_INIT_FUNC_TRACE(); 3104 3105 txq = dev->data->tx_queues[tx_queue_id]; 3106 3107 /* Wait until Tx queue is empty */ 3108 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3109 do { 3110 rte_delay_us(RTE_NGBE_WAIT_100_US); 3111 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx)); 3112 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx)); 3113 } while (--poll_ms && (txtdh != txtdt)); 3114 if (poll_ms == 0) 3115 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.", 3116 tx_queue_id); 3117 3118 ngbe_dev_save_tx_queue(hw, txq->reg_idx); 3119 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0); 3120 3121 /* Wait until Tx Enable bit clear */ 3122 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3123 do { 3124 rte_delay_ms(1); 3125 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3126 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA)); 3127 if (poll_ms == 0) 3128 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 3129 tx_queue_id); 3130 3131 rte_delay_us(RTE_NGBE_WAIT_100_US); 3132 ngbe_dev_store_tx_queue(hw, txq->reg_idx); 3133 3134 if (txq->ops != NULL) { 3135 txq->ops->release_mbufs(txq); 3136 txq->ops->reset(txq); 3137 } 3138 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3139 3140 return 0; 3141 } 3142