1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdint.h> 9 #include <rte_ethdev.h> 10 #include <ethdev_driver.h> 11 #include <rte_malloc.h> 12 #include <rte_net.h> 13 14 #include "ngbe_logs.h" 15 #include "base/ngbe.h" 16 #include "ngbe_ethdev.h" 17 #include "ngbe_rxtx.h" 18 19 #ifdef RTE_LIBRTE_IEEE1588 20 #define NGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST 21 #else 22 #define NGBE_TX_IEEE1588_TMST 0 23 #endif 24 25 /* Bit Mask to indicate what bits required for building Tx context */ 26 static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM | 27 RTE_MBUF_F_TX_IPV6 | 28 RTE_MBUF_F_TX_IPV4 | 29 RTE_MBUF_F_TX_VLAN | 30 RTE_MBUF_F_TX_L4_MASK | 31 RTE_MBUF_F_TX_TCP_SEG | 32 NGBE_TX_IEEE1588_TMST); 33 34 #define NGBE_TX_OFFLOAD_NOTSUP_MASK \ 35 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ NGBE_TX_OFFLOAD_MASK) 36 37 /* 38 * Prefetch a cache line into all cache levels. 39 */ 40 #define rte_ngbe_prefetch(p) rte_prefetch0(p) 41 42 /********************************************************************* 43 * 44 * Tx functions 45 * 46 **********************************************************************/ 47 48 /* 49 * Check for descriptors with their DD bit set and free mbufs. 50 * Return the total number of buffers freed. 51 */ 52 static __rte_always_inline int 53 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq) 54 { 55 struct ngbe_tx_entry *txep; 56 uint32_t status; 57 int i, nb_free = 0; 58 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ]; 59 60 /* check DD bit on threshold descriptor */ 61 status = txq->tx_ring[txq->tx_next_dd].dw3; 62 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 63 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 64 ngbe_set32_masked(txq->tdc_reg_addr, 65 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 66 return 0; 67 } 68 69 /* 70 * first buffer to free from S/W ring is at index 71 * tx_next_dd - (tx_free_thresh-1) 72 */ 73 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 74 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 75 /* free buffers one at a time */ 76 m = rte_pktmbuf_prefree_seg(txep->mbuf); 77 txep->mbuf = NULL; 78 79 if (unlikely(m == NULL)) 80 continue; 81 82 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ || 83 (nb_free > 0 && m->pool != free[0]->pool)) { 84 rte_mempool_put_bulk(free[0]->pool, 85 (void **)free, nb_free); 86 nb_free = 0; 87 } 88 89 free[nb_free++] = m; 90 } 91 92 if (nb_free > 0) 93 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 94 95 /* buffers were freed, update counters */ 96 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 97 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 98 if (txq->tx_next_dd >= txq->nb_tx_desc) 99 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 100 101 return txq->tx_free_thresh; 102 } 103 104 /* Populate 4 descriptors with data from 4 mbufs */ 105 static inline void 106 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 107 { 108 uint64_t buf_dma_addr; 109 uint32_t pkt_len; 110 int i; 111 112 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 113 buf_dma_addr = rte_mbuf_data_iova(*pkts); 114 pkt_len = (*pkts)->data_len; 115 116 /* write data to descriptor */ 117 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 118 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 119 NGBE_TXD_DATLEN(pkt_len)); 120 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 121 122 rte_prefetch0(&(*pkts)->pool); 123 } 124 } 125 126 /* Populate 1 descriptor with data from 1 mbuf */ 127 static inline void 128 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 129 { 130 uint64_t buf_dma_addr; 131 uint32_t pkt_len; 132 133 buf_dma_addr = rte_mbuf_data_iova(*pkts); 134 pkt_len = (*pkts)->data_len; 135 136 /* write data to descriptor */ 137 txdp->qw0 = cpu_to_le64(buf_dma_addr); 138 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 139 NGBE_TXD_DATLEN(pkt_len)); 140 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 141 142 rte_prefetch0(&(*pkts)->pool); 143 } 144 145 /* 146 * Fill H/W descriptor ring with mbuf data. 147 * Copy mbuf pointers to the S/W ring. 148 */ 149 static inline void 150 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts, 151 uint16_t nb_pkts) 152 { 153 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 154 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 155 const int N_PER_LOOP = 4; 156 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 157 int mainpart, leftover; 158 int i, j; 159 160 /* 161 * Process most of the packets in chunks of N pkts. Any 162 * leftover packets will get processed one at a time. 163 */ 164 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 165 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 166 for (i = 0; i < mainpart; i += N_PER_LOOP) { 167 /* Copy N mbuf pointers to the S/W ring */ 168 for (j = 0; j < N_PER_LOOP; ++j) 169 (txep + i + j)->mbuf = *(pkts + i + j); 170 tx4(txdp + i, pkts + i); 171 } 172 173 if (unlikely(leftover > 0)) { 174 for (i = 0; i < leftover; ++i) { 175 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 176 tx1(txdp + mainpart + i, pkts + mainpart + i); 177 } 178 } 179 } 180 181 static inline uint16_t 182 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 183 uint16_t nb_pkts) 184 { 185 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 186 uint16_t n = 0; 187 188 /* 189 * Begin scanning the H/W ring for done descriptors when the 190 * number of available descriptors drops below tx_free_thresh. 191 * For each done descriptor, free the associated buffer. 192 */ 193 if (txq->nb_tx_free < txq->tx_free_thresh) 194 ngbe_tx_free_bufs(txq); 195 196 /* Only use descriptors that are available */ 197 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 198 if (unlikely(nb_pkts == 0)) 199 return 0; 200 201 /* Use exactly nb_pkts descriptors */ 202 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 203 204 /* 205 * At this point, we know there are enough descriptors in the 206 * ring to transmit all the packets. This assumes that each 207 * mbuf contains a single segment, and that no new offloads 208 * are expected, which would require a new context descriptor. 209 */ 210 211 /* 212 * See if we're going to wrap-around. If so, handle the top 213 * of the descriptor ring first, then do the bottom. If not, 214 * the processing looks just like the "bottom" part anyway... 215 */ 216 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 217 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 218 ngbe_tx_fill_hw_ring(txq, tx_pkts, n); 219 txq->tx_tail = 0; 220 } 221 222 /* Fill H/W descriptor ring with mbuf data */ 223 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 224 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 225 226 /* 227 * Check for wrap-around. This would only happen if we used 228 * up to the last descriptor in the ring, no more, no less. 229 */ 230 if (txq->tx_tail >= txq->nb_tx_desc) 231 txq->tx_tail = 0; 232 233 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 234 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 235 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 236 237 /* update tail pointer */ 238 rte_wmb(); 239 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 240 241 return nb_pkts; 242 } 243 244 uint16_t 245 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 246 uint16_t nb_pkts) 247 { 248 uint16_t nb_tx; 249 250 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 251 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST)) 252 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 253 254 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 255 nb_tx = 0; 256 while (nb_pkts != 0) { 257 uint16_t ret, n; 258 259 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST); 260 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 261 nb_tx = (uint16_t)(nb_tx + ret); 262 nb_pkts = (uint16_t)(nb_pkts - ret); 263 if (ret < n) 264 break; 265 } 266 267 return nb_tx; 268 } 269 270 static inline void 271 ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq, 272 volatile struct ngbe_tx_ctx_desc *ctx_txd, 273 uint64_t ol_flags, union ngbe_tx_offload tx_offload) 274 { 275 union ngbe_tx_offload tx_offload_mask; 276 uint32_t type_tucmd_mlhl; 277 uint32_t mss_l4len_idx; 278 uint32_t ctx_idx; 279 uint32_t vlan_macip_lens; 280 uint32_t tunnel_seed; 281 282 ctx_idx = txq->ctx_curr; 283 tx_offload_mask.data[0] = 0; 284 tx_offload_mask.data[1] = 0; 285 286 /* Specify which HW CTX to upload. */ 287 mss_l4len_idx = NGBE_TXD_IDX(ctx_idx); 288 type_tucmd_mlhl = NGBE_TXD_CTXT; 289 290 tx_offload_mask.ptid |= ~0; 291 type_tucmd_mlhl |= NGBE_TXD_PTID(tx_offload.ptid); 292 293 /* check if TCP segmentation required for this packet */ 294 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 295 tx_offload_mask.l2_len |= ~0; 296 tx_offload_mask.l3_len |= ~0; 297 tx_offload_mask.l4_len |= ~0; 298 tx_offload_mask.tso_segsz |= ~0; 299 mss_l4len_idx |= NGBE_TXD_MSS(tx_offload.tso_segsz); 300 mss_l4len_idx |= NGBE_TXD_L4LEN(tx_offload.l4_len); 301 } else { /* no TSO, check if hardware checksum is needed */ 302 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 303 tx_offload_mask.l2_len |= ~0; 304 tx_offload_mask.l3_len |= ~0; 305 } 306 307 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 308 case RTE_MBUF_F_TX_UDP_CKSUM: 309 mss_l4len_idx |= 310 NGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr)); 311 tx_offload_mask.l2_len |= ~0; 312 tx_offload_mask.l3_len |= ~0; 313 break; 314 case RTE_MBUF_F_TX_TCP_CKSUM: 315 mss_l4len_idx |= 316 NGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr)); 317 tx_offload_mask.l2_len |= ~0; 318 tx_offload_mask.l3_len |= ~0; 319 break; 320 case RTE_MBUF_F_TX_SCTP_CKSUM: 321 mss_l4len_idx |= 322 NGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr)); 323 tx_offload_mask.l2_len |= ~0; 324 tx_offload_mask.l3_len |= ~0; 325 break; 326 default: 327 break; 328 } 329 } 330 331 vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1); 332 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len); 333 334 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 335 tx_offload_mask.vlan_tci |= ~0; 336 vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci); 337 } 338 339 tunnel_seed = 0; 340 341 txq->ctx_cache[ctx_idx].flags = ol_flags; 342 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 343 tx_offload_mask.data[0] & tx_offload.data[0]; 344 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 345 tx_offload_mask.data[1] & tx_offload.data[1]; 346 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 347 348 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens); 349 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed); 350 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl); 351 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx); 352 } 353 354 /* 355 * Check which hardware context can be used. Use the existing match 356 * or create a new context descriptor. 357 */ 358 static inline uint32_t 359 what_ctx_update(struct ngbe_tx_queue *txq, uint64_t flags, 360 union ngbe_tx_offload tx_offload) 361 { 362 /* If match with the current used context */ 363 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 364 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 365 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 366 & tx_offload.data[0])) && 367 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 368 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 369 & tx_offload.data[1])))) 370 return txq->ctx_curr; 371 372 /* What if match with the next context */ 373 txq->ctx_curr ^= 1; 374 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 375 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 376 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 377 & tx_offload.data[0])) && 378 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 379 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 380 & tx_offload.data[1])))) 381 return txq->ctx_curr; 382 383 /* Mismatch, use the previous context */ 384 return NGBE_CTX_NUM; 385 } 386 387 static inline uint32_t 388 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 389 { 390 uint32_t tmp = 0; 391 392 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) { 393 tmp |= NGBE_TXD_CC; 394 tmp |= NGBE_TXD_L4CS; 395 } 396 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 397 tmp |= NGBE_TXD_CC; 398 tmp |= NGBE_TXD_IPCS; 399 } 400 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 401 tmp |= NGBE_TXD_CC; 402 tmp |= NGBE_TXD_EIPCS; 403 } 404 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 405 tmp |= NGBE_TXD_CC; 406 /* implies IPv4 cksum */ 407 if (ol_flags & RTE_MBUF_F_TX_IPV4) 408 tmp |= NGBE_TXD_IPCS; 409 tmp |= NGBE_TXD_L4CS; 410 } 411 if (ol_flags & RTE_MBUF_F_TX_VLAN) 412 tmp |= NGBE_TXD_CC; 413 414 return tmp; 415 } 416 417 static inline uint32_t 418 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 419 { 420 uint32_t cmdtype = 0; 421 422 if (ol_flags & RTE_MBUF_F_TX_VLAN) 423 cmdtype |= NGBE_TXD_VLE; 424 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 425 cmdtype |= NGBE_TXD_TSE; 426 return cmdtype; 427 } 428 429 static inline uint32_t 430 tx_desc_ol_flags_to_ptype(uint64_t oflags) 431 { 432 uint32_t ptype; 433 434 /* L2 level */ 435 ptype = RTE_PTYPE_L2_ETHER; 436 if (oflags & RTE_MBUF_F_TX_VLAN) 437 ptype |= RTE_PTYPE_L2_ETHER_VLAN; 438 439 /* L3 level */ 440 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM)) 441 ptype |= RTE_PTYPE_L3_IPV4; 442 else if (oflags & (RTE_MBUF_F_TX_IPV6)) 443 ptype |= RTE_PTYPE_L3_IPV6; 444 445 /* L4 level */ 446 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) { 447 case RTE_MBUF_F_TX_TCP_CKSUM: 448 ptype |= RTE_PTYPE_L4_TCP; 449 break; 450 case RTE_MBUF_F_TX_UDP_CKSUM: 451 ptype |= RTE_PTYPE_L4_UDP; 452 break; 453 case RTE_MBUF_F_TX_SCTP_CKSUM: 454 ptype |= RTE_PTYPE_L4_SCTP; 455 break; 456 } 457 458 if (oflags & RTE_MBUF_F_TX_TCP_SEG) 459 ptype |= RTE_PTYPE_L4_TCP; 460 461 return ptype; 462 } 463 464 static inline uint8_t 465 tx_desc_ol_flags_to_ptid(uint64_t oflags) 466 { 467 uint32_t ptype; 468 469 ptype = tx_desc_ol_flags_to_ptype(oflags); 470 471 return ngbe_encode_ptype(ptype); 472 } 473 474 /* Reset transmit descriptors after they have been used */ 475 static inline int 476 ngbe_xmit_cleanup(struct ngbe_tx_queue *txq) 477 { 478 struct ngbe_tx_entry *sw_ring = txq->sw_ring; 479 volatile struct ngbe_tx_desc *txr = txq->tx_ring; 480 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 481 uint16_t nb_tx_desc = txq->nb_tx_desc; 482 uint16_t desc_to_clean_to; 483 uint16_t nb_tx_to_clean; 484 uint32_t status; 485 486 /* Determine the last descriptor needing to be cleaned */ 487 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh); 488 if (desc_to_clean_to >= nb_tx_desc) 489 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 490 491 /* Check to make sure the last descriptor to clean is done */ 492 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 493 status = txr[desc_to_clean_to].dw3; 494 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 495 PMD_TX_LOG(DEBUG, 496 "Tx descriptor %4u is not done" 497 "(port=%d queue=%d)", 498 desc_to_clean_to, 499 txq->port_id, txq->queue_id); 500 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 501 ngbe_set32_masked(txq->tdc_reg_addr, 502 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 503 /* Failed to clean any descriptors, better luck next time */ 504 return -(1); 505 } 506 507 /* Figure out how many descriptors will be cleaned */ 508 if (last_desc_cleaned > desc_to_clean_to) 509 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 510 desc_to_clean_to); 511 else 512 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 513 last_desc_cleaned); 514 515 PMD_TX_LOG(DEBUG, 516 "Cleaning %4u Tx descriptors: %4u to %4u (port=%d queue=%d)", 517 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 518 txq->port_id, txq->queue_id); 519 520 /* 521 * The last descriptor to clean is done, so that means all the 522 * descriptors from the last descriptor that was cleaned 523 * up to the last descriptor with the RS bit set 524 * are done. Only reset the threshold descriptor. 525 */ 526 txr[desc_to_clean_to].dw3 = 0; 527 528 /* Update the txq to reflect the last descriptor that was cleaned */ 529 txq->last_desc_cleaned = desc_to_clean_to; 530 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 531 532 /* No Error */ 533 return 0; 534 } 535 536 uint16_t 537 ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 538 uint16_t nb_pkts) 539 { 540 struct ngbe_tx_queue *txq; 541 struct ngbe_tx_entry *sw_ring; 542 struct ngbe_tx_entry *txe, *txn; 543 volatile struct ngbe_tx_desc *txr; 544 volatile struct ngbe_tx_desc *txd; 545 struct rte_mbuf *tx_pkt; 546 struct rte_mbuf *m_seg; 547 uint64_t buf_dma_addr; 548 uint32_t olinfo_status; 549 uint32_t cmd_type_len; 550 uint32_t pkt_len; 551 uint16_t slen; 552 uint64_t ol_flags; 553 uint16_t tx_id; 554 uint16_t tx_last; 555 uint16_t nb_tx; 556 uint16_t nb_used; 557 uint64_t tx_ol_req; 558 uint32_t ctx = 0; 559 uint32_t new_ctx; 560 union ngbe_tx_offload tx_offload; 561 562 tx_offload.data[0] = 0; 563 tx_offload.data[1] = 0; 564 txq = tx_queue; 565 sw_ring = txq->sw_ring; 566 txr = txq->tx_ring; 567 tx_id = txq->tx_tail; 568 txe = &sw_ring[tx_id]; 569 570 /* Determine if the descriptor ring needs to be cleaned. */ 571 if (txq->nb_tx_free < txq->tx_free_thresh) 572 ngbe_xmit_cleanup(txq); 573 574 rte_prefetch0(&txe->mbuf->pool); 575 576 /* Tx loop */ 577 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 578 new_ctx = 0; 579 tx_pkt = *tx_pkts++; 580 pkt_len = tx_pkt->pkt_len; 581 582 /* 583 * Determine how many (if any) context descriptors 584 * are needed for offload functionality. 585 */ 586 ol_flags = tx_pkt->ol_flags; 587 588 /* If hardware offload required */ 589 tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK; 590 if (tx_ol_req) { 591 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req); 592 tx_offload.l2_len = tx_pkt->l2_len; 593 tx_offload.l3_len = tx_pkt->l3_len; 594 tx_offload.l4_len = tx_pkt->l4_len; 595 tx_offload.vlan_tci = tx_pkt->vlan_tci; 596 tx_offload.tso_segsz = tx_pkt->tso_segsz; 597 598 /* If new context need be built or reuse the exist ctx*/ 599 ctx = what_ctx_update(txq, tx_ol_req, tx_offload); 600 /* Only allocate context descriptor if required */ 601 new_ctx = (ctx == NGBE_CTX_NUM); 602 ctx = txq->ctx_curr; 603 } 604 605 /* 606 * Keep track of how many descriptors are used this loop 607 * This will always be the number of segments + the number of 608 * Context descriptors required to transmit the packet 609 */ 610 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 611 612 /* 613 * The number of descriptors that must be allocated for a 614 * packet is the number of segments of that packet, plus 1 615 * Context Descriptor for the hardware offload, if any. 616 * Determine the last Tx descriptor to allocate in the Tx ring 617 * for the packet, starting from the current position (tx_id) 618 * in the ring. 619 */ 620 tx_last = (uint16_t)(tx_id + nb_used - 1); 621 622 /* Circular ring */ 623 if (tx_last >= txq->nb_tx_desc) 624 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 625 626 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 627 " tx_first=%u tx_last=%u", 628 (uint16_t)txq->port_id, 629 (uint16_t)txq->queue_id, 630 (uint32_t)pkt_len, 631 (uint16_t)tx_id, 632 (uint16_t)tx_last); 633 634 /* 635 * Make sure there are enough Tx descriptors available to 636 * transmit the entire packet. 637 * nb_used better be less than or equal to txq->tx_free_thresh 638 */ 639 if (nb_used > txq->nb_tx_free) { 640 PMD_TX_LOG(DEBUG, 641 "Not enough free Tx descriptors " 642 "nb_used=%4u nb_free=%4u " 643 "(port=%d queue=%d)", 644 nb_used, txq->nb_tx_free, 645 txq->port_id, txq->queue_id); 646 647 if (ngbe_xmit_cleanup(txq) != 0) { 648 /* Could not clean any descriptors */ 649 if (nb_tx == 0) 650 return 0; 651 goto end_of_tx; 652 } 653 654 /* nb_used better be <= txq->tx_free_thresh */ 655 if (unlikely(nb_used > txq->tx_free_thresh)) { 656 PMD_TX_LOG(DEBUG, 657 "The number of descriptors needed to " 658 "transmit the packet exceeds the " 659 "RS bit threshold. This will impact " 660 "performance." 661 "nb_used=%4u nb_free=%4u " 662 "tx_free_thresh=%4u. " 663 "(port=%d queue=%d)", 664 nb_used, txq->nb_tx_free, 665 txq->tx_free_thresh, 666 txq->port_id, txq->queue_id); 667 /* 668 * Loop here until there are enough Tx 669 * descriptors or until the ring cannot be 670 * cleaned. 671 */ 672 while (nb_used > txq->nb_tx_free) { 673 if (ngbe_xmit_cleanup(txq) != 0) { 674 /* 675 * Could not clean any 676 * descriptors 677 */ 678 if (nb_tx == 0) 679 return 0; 680 goto end_of_tx; 681 } 682 } 683 } 684 } 685 686 /* 687 * By now there are enough free Tx descriptors to transmit 688 * the packet. 689 */ 690 691 /* 692 * Set common flags of all Tx Data Descriptors. 693 * 694 * The following bits must be set in the first Data Descriptor 695 * and are ignored in the other ones: 696 * - NGBE_TXD_FCS 697 * 698 * The following bits must only be set in the last Data 699 * Descriptor: 700 * - NGBE_TXD_EOP 701 */ 702 cmd_type_len = NGBE_TXD_FCS; 703 704 #ifdef RTE_LIBRTE_IEEE1588 705 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 706 cmd_type_len |= NGBE_TXD_1588; 707 #endif 708 709 olinfo_status = 0; 710 if (tx_ol_req) { 711 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 712 /* when TSO is on, paylen in descriptor is the 713 * not the packet len but the tcp payload len 714 */ 715 pkt_len -= (tx_offload.l2_len + 716 tx_offload.l3_len + tx_offload.l4_len); 717 } 718 719 /* 720 * Setup the Tx Context Descriptor if required 721 */ 722 if (new_ctx) { 723 volatile struct ngbe_tx_ctx_desc *ctx_txd; 724 725 ctx_txd = (volatile struct ngbe_tx_ctx_desc *) 726 &txr[tx_id]; 727 728 txn = &sw_ring[txe->next_id]; 729 rte_prefetch0(&txn->mbuf->pool); 730 731 if (txe->mbuf != NULL) { 732 rte_pktmbuf_free_seg(txe->mbuf); 733 txe->mbuf = NULL; 734 } 735 736 ngbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 737 tx_offload); 738 739 txe->last_id = tx_last; 740 tx_id = txe->next_id; 741 txe = txn; 742 } 743 744 /* 745 * Setup the Tx Data Descriptor, 746 * This path will go through 747 * whatever new/reuse the context descriptor 748 */ 749 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 750 olinfo_status |= 751 tx_desc_cksum_flags_to_olinfo(ol_flags); 752 olinfo_status |= NGBE_TXD_IDX(ctx); 753 } 754 755 olinfo_status |= NGBE_TXD_PAYLEN(pkt_len); 756 757 m_seg = tx_pkt; 758 do { 759 txd = &txr[tx_id]; 760 txn = &sw_ring[txe->next_id]; 761 rte_prefetch0(&txn->mbuf->pool); 762 763 if (txe->mbuf != NULL) 764 rte_pktmbuf_free_seg(txe->mbuf); 765 txe->mbuf = m_seg; 766 767 /* 768 * Set up Transmit Data Descriptor. 769 */ 770 slen = m_seg->data_len; 771 buf_dma_addr = rte_mbuf_data_iova(m_seg); 772 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr); 773 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen); 774 txd->dw3 = rte_cpu_to_le_32(olinfo_status); 775 txe->last_id = tx_last; 776 tx_id = txe->next_id; 777 txe = txn; 778 m_seg = m_seg->next; 779 } while (m_seg != NULL); 780 781 /* 782 * The last packet data descriptor needs End Of Packet (EOP) 783 */ 784 cmd_type_len |= NGBE_TXD_EOP; 785 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 786 787 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len); 788 } 789 790 end_of_tx: 791 792 rte_wmb(); 793 794 /* 795 * Set the Transmit Descriptor Tail (TDT) 796 */ 797 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 798 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 799 (uint16_t)tx_id, (uint16_t)nb_tx); 800 ngbe_set32_relaxed(txq->tdt_reg_addr, tx_id); 801 txq->tx_tail = tx_id; 802 803 return nb_tx; 804 } 805 806 /********************************************************************* 807 * 808 * Tx prep functions 809 * 810 **********************************************************************/ 811 uint16_t 812 ngbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 813 { 814 int i, ret; 815 uint64_t ol_flags; 816 struct rte_mbuf *m; 817 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 818 819 for (i = 0; i < nb_pkts; i++) { 820 m = tx_pkts[i]; 821 ol_flags = m->ol_flags; 822 823 /** 824 * Check if packet meets requirements for number of segments 825 * 826 * NOTE: for ngbe it's always (40 - WTHRESH) for both TSO and 827 * non-TSO 828 */ 829 830 if (m->nb_segs > NGBE_TX_MAX_SEG - txq->wthresh) { 831 rte_errno = -EINVAL; 832 return i; 833 } 834 835 if (ol_flags & NGBE_TX_OFFLOAD_NOTSUP_MASK) { 836 rte_errno = -ENOTSUP; 837 return i; 838 } 839 840 #ifdef RTE_ETHDEV_DEBUG_TX 841 ret = rte_validate_tx_offload(m); 842 if (ret != 0) { 843 rte_errno = ret; 844 return i; 845 } 846 #endif 847 ret = rte_net_intel_cksum_prepare(m); 848 if (ret != 0) { 849 rte_errno = ret; 850 return i; 851 } 852 } 853 854 return i; 855 } 856 857 /********************************************************************* 858 * 859 * Rx functions 860 * 861 **********************************************************************/ 862 static inline uint32_t 863 ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) 864 { 865 uint16_t ptid = NGBE_RXD_PTID(pkt_info); 866 867 ptid &= ptid_mask; 868 869 return ngbe_decode_ptype(ptid); 870 } 871 872 static inline uint64_t 873 ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) 874 { 875 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { 876 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 877 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 878 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 879 0, 0, 0, RTE_MBUF_F_RX_FDIR, 880 }; 881 #ifdef RTE_LIBRTE_IEEE1588 882 static uint64_t ip_pkt_etqf_map[8] = { 883 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP, 884 0, 0, 0, 0, 885 }; 886 int etfid = ngbe_etflt_id(NGBE_RXD_PTID(pkt_info)); 887 if (likely(-1 != etfid)) 888 return ip_pkt_etqf_map[etfid] | 889 ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 890 else 891 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 892 #else 893 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 894 #endif 895 } 896 897 static inline uint64_t 898 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 899 { 900 uint64_t pkt_flags; 901 902 /* 903 * Check if VLAN present only. 904 * Do not check whether L3/L4 rx checksum done by NIC or not, 905 * That can be found from rte_eth_rxmode.offloads flag 906 */ 907 pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN && 908 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED) 909 ? vlan_flags : 0; 910 911 #ifdef RTE_LIBRTE_IEEE1588 912 if (rx_status & NGBE_RXD_STAT_1588) 913 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST; 914 #endif 915 return pkt_flags; 916 } 917 918 static inline uint64_t 919 rx_desc_error_to_pkt_flags(uint32_t rx_status) 920 { 921 uint64_t pkt_flags = 0; 922 923 /* checksum offload can't be disabled */ 924 if (rx_status & NGBE_RXD_STAT_IPCS) 925 pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS 926 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD); 927 928 if (rx_status & NGBE_RXD_STAT_L4CS) 929 pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS 930 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD); 931 932 if (rx_status & NGBE_RXD_STAT_EIPCS && 933 rx_status & NGBE_RXD_ERR_EIPCS) 934 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 935 936 return pkt_flags; 937 } 938 939 /* 940 * LOOK_AHEAD defines how many desc statuses to check beyond the 941 * current descriptor. 942 * It must be a pound define for optimal performance. 943 * Do not change the value of LOOK_AHEAD, as the ngbe_rx_scan_hw_ring 944 * function only works with LOOK_AHEAD=8. 945 */ 946 #define LOOK_AHEAD 8 947 #if (LOOK_AHEAD != 8) 948 #error "PMD NGBE: LOOK_AHEAD must be 8\n" 949 #endif 950 static inline int 951 ngbe_rx_scan_hw_ring(struct ngbe_rx_queue *rxq) 952 { 953 volatile struct ngbe_rx_desc *rxdp; 954 struct ngbe_rx_entry *rxep; 955 struct rte_mbuf *mb; 956 uint16_t pkt_len; 957 uint64_t pkt_flags; 958 int nb_dd; 959 uint32_t s[LOOK_AHEAD]; 960 uint32_t pkt_info[LOOK_AHEAD]; 961 int i, j, nb_rx = 0; 962 uint32_t status; 963 964 /* get references to current descriptor and S/W ring entry */ 965 rxdp = &rxq->rx_ring[rxq->rx_tail]; 966 rxep = &rxq->sw_ring[rxq->rx_tail]; 967 968 status = rxdp->qw1.lo.status; 969 /* check to make sure there is at least 1 packet to receive */ 970 if (!(status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 971 return 0; 972 973 /* 974 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 975 * reference packets that are ready to be received. 976 */ 977 for (i = 0; i < RTE_PMD_NGBE_RX_MAX_BURST; 978 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 979 /* Read desc statuses backwards to avoid race condition */ 980 for (j = 0; j < LOOK_AHEAD; j++) 981 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); 982 983 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 984 985 /* Compute how many status bits were set */ 986 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 987 (s[nb_dd] & NGBE_RXD_STAT_DD); nb_dd++) 988 ; 989 990 for (j = 0; j < nb_dd; j++) 991 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0); 992 993 nb_rx += nb_dd; 994 995 /* Translate descriptor info to mbuf format */ 996 for (j = 0; j < nb_dd; ++j) { 997 mb = rxep[j].mbuf; 998 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) - 999 rxq->crc_len; 1000 mb->data_len = pkt_len; 1001 mb->pkt_len = pkt_len; 1002 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag); 1003 1004 /* convert descriptor fields to rte mbuf flags */ 1005 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1006 rxq->vlan_flags); 1007 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]); 1008 pkt_flags |= 1009 ngbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]); 1010 mb->ol_flags = pkt_flags; 1011 mb->packet_type = 1012 ngbe_rxd_pkt_info_to_pkt_type(pkt_info[j], 1013 NGBE_PTID_MASK); 1014 1015 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1016 mb->hash.rss = 1017 rte_le_to_cpu_32(rxdp[j].qw0.dw1); 1018 } 1019 1020 /* Move mbuf pointers from the S/W ring to the stage */ 1021 for (j = 0; j < LOOK_AHEAD; ++j) 1022 rxq->rx_stage[i + j] = rxep[j].mbuf; 1023 1024 /* stop if all requested packets could not be received */ 1025 if (nb_dd != LOOK_AHEAD) 1026 break; 1027 } 1028 1029 /* clear software ring entries so we can cleanup correctly */ 1030 for (i = 0; i < nb_rx; ++i) 1031 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1032 1033 return nb_rx; 1034 } 1035 1036 static inline int 1037 ngbe_rx_alloc_bufs(struct ngbe_rx_queue *rxq, bool reset_mbuf) 1038 { 1039 volatile struct ngbe_rx_desc *rxdp; 1040 struct ngbe_rx_entry *rxep; 1041 struct rte_mbuf *mb; 1042 uint16_t alloc_idx; 1043 __le64 dma_addr; 1044 int diag, i; 1045 1046 /* allocate buffers in bulk directly into the S/W ring */ 1047 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1048 rxep = &rxq->sw_ring[alloc_idx]; 1049 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1050 rxq->rx_free_thresh); 1051 if (unlikely(diag != 0)) 1052 return -ENOMEM; 1053 1054 rxdp = &rxq->rx_ring[alloc_idx]; 1055 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1056 /* populate the static rte mbuf fields */ 1057 mb = rxep[i].mbuf; 1058 if (reset_mbuf) 1059 mb->port = rxq->port_id; 1060 1061 rte_mbuf_refcnt_set(mb, 1); 1062 mb->data_off = RTE_PKTMBUF_HEADROOM; 1063 1064 /* populate the descriptors */ 1065 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1066 NGBE_RXD_HDRADDR(&rxdp[i], 0); 1067 NGBE_RXD_PKTADDR(&rxdp[i], dma_addr); 1068 } 1069 1070 /* update state of internal queue structure */ 1071 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1072 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1073 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1074 1075 /* no errors */ 1076 return 0; 1077 } 1078 1079 static inline uint16_t 1080 ngbe_rx_fill_from_stage(struct ngbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1081 uint16_t nb_pkts) 1082 { 1083 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1084 int i; 1085 1086 /* how many packets are ready to return? */ 1087 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1088 1089 /* copy mbuf pointers to the application's packet list */ 1090 for (i = 0; i < nb_pkts; ++i) 1091 rx_pkts[i] = stage[i]; 1092 1093 /* update internal queue state */ 1094 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1095 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1096 1097 return nb_pkts; 1098 } 1099 1100 static inline uint16_t 1101 ngbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1102 uint16_t nb_pkts) 1103 { 1104 struct ngbe_rx_queue *rxq = (struct ngbe_rx_queue *)rx_queue; 1105 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1106 uint16_t nb_rx = 0; 1107 1108 /* Any previously recv'd pkts will be returned from the Rx stage */ 1109 if (rxq->rx_nb_avail) 1110 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1111 1112 /* Scan the H/W ring for packets to receive */ 1113 nb_rx = (uint16_t)ngbe_rx_scan_hw_ring(rxq); 1114 1115 /* update internal queue state */ 1116 rxq->rx_next_avail = 0; 1117 rxq->rx_nb_avail = nb_rx; 1118 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1119 1120 /* if required, allocate new buffers to replenish descriptors */ 1121 if (rxq->rx_tail > rxq->rx_free_trigger) { 1122 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1123 1124 if (ngbe_rx_alloc_bufs(rxq, true) != 0) { 1125 int i, j; 1126 1127 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1128 "queue_id=%u", (uint16_t)rxq->port_id, 1129 (uint16_t)rxq->queue_id); 1130 1131 dev->data->rx_mbuf_alloc_failed += 1132 rxq->rx_free_thresh; 1133 1134 /* 1135 * Need to rewind any previous receives if we cannot 1136 * allocate new buffers to replenish the old ones. 1137 */ 1138 rxq->rx_nb_avail = 0; 1139 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1140 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1141 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1142 1143 return 0; 1144 } 1145 1146 /* update tail pointer */ 1147 rte_wmb(); 1148 ngbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger); 1149 } 1150 1151 if (rxq->rx_tail >= rxq->nb_rx_desc) 1152 rxq->rx_tail = 0; 1153 1154 /* received any packets this loop? */ 1155 if (rxq->rx_nb_avail) 1156 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1157 1158 return 0; 1159 } 1160 1161 /* split requests into chunks of size RTE_PMD_NGBE_RX_MAX_BURST */ 1162 uint16_t 1163 ngbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1164 uint16_t nb_pkts) 1165 { 1166 uint16_t nb_rx; 1167 1168 if (unlikely(nb_pkts == 0)) 1169 return 0; 1170 1171 if (likely(nb_pkts <= RTE_PMD_NGBE_RX_MAX_BURST)) 1172 return ngbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1173 1174 /* request is relatively large, chunk it up */ 1175 nb_rx = 0; 1176 while (nb_pkts) { 1177 uint16_t ret, n; 1178 1179 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_RX_MAX_BURST); 1180 ret = ngbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1181 nb_rx = (uint16_t)(nb_rx + ret); 1182 nb_pkts = (uint16_t)(nb_pkts - ret); 1183 if (ret < n) 1184 break; 1185 } 1186 1187 return nb_rx; 1188 } 1189 1190 uint16_t 1191 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1192 uint16_t nb_pkts) 1193 { 1194 struct ngbe_rx_queue *rxq; 1195 volatile struct ngbe_rx_desc *rx_ring; 1196 volatile struct ngbe_rx_desc *rxdp; 1197 struct ngbe_rx_entry *sw_ring; 1198 struct ngbe_rx_entry *rxe; 1199 struct rte_mbuf *rxm; 1200 struct rte_mbuf *nmb; 1201 struct ngbe_rx_desc rxd; 1202 uint64_t dma_addr; 1203 uint32_t staterr; 1204 uint32_t pkt_info; 1205 uint16_t pkt_len; 1206 uint16_t rx_id; 1207 uint16_t nb_rx; 1208 uint16_t nb_hold; 1209 uint64_t pkt_flags; 1210 1211 nb_rx = 0; 1212 nb_hold = 0; 1213 rxq = rx_queue; 1214 rx_id = rxq->rx_tail; 1215 rx_ring = rxq->rx_ring; 1216 sw_ring = rxq->sw_ring; 1217 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1218 while (nb_rx < nb_pkts) { 1219 /* 1220 * The order of operations here is important as the DD status 1221 * bit must not be read after any other descriptor fields. 1222 * rx_ring and rxdp are pointing to volatile data so the order 1223 * of accesses cannot be reordered by the compiler. If they were 1224 * not volatile, they could be reordered which could lead to 1225 * using invalid descriptor fields when read from rxd. 1226 */ 1227 rxdp = &rx_ring[rx_id]; 1228 staterr = rxdp->qw1.lo.status; 1229 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1230 break; 1231 rxd = *rxdp; 1232 1233 /* 1234 * End of packet. 1235 * 1236 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet 1237 * is likely to be invalid and to be dropped by the various 1238 * validation checks performed by the network stack. 1239 * 1240 * Allocate a new mbuf to replenish the RX ring descriptor. 1241 * If the allocation fails: 1242 * - arrange for that Rx descriptor to be the first one 1243 * being parsed the next time the receive function is 1244 * invoked [on the same queue]. 1245 * 1246 * - Stop parsing the Rx ring and return immediately. 1247 * 1248 * This policy do not drop the packet received in the Rx 1249 * descriptor for which the allocation of a new mbuf failed. 1250 * Thus, it allows that packet to be later retrieved if 1251 * mbuf have been freed in the mean time. 1252 * As a side effect, holding Rx descriptors instead of 1253 * systematically giving them back to the NIC may lead to 1254 * Rx ring exhaustion situations. 1255 * However, the NIC can gracefully prevent such situations 1256 * to happen by sending specific "back-pressure" flow control 1257 * frames to its peer(s). 1258 */ 1259 PMD_RX_LOG(DEBUG, 1260 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u", 1261 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1262 (uint16_t)rx_id, (uint32_t)staterr, 1263 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 1264 1265 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1266 if (nmb == NULL) { 1267 PMD_RX_LOG(DEBUG, 1268 "Rx mbuf alloc failed port_id=%u queue_id=%u", 1269 (uint16_t)rxq->port_id, 1270 (uint16_t)rxq->queue_id); 1271 dev->data->rx_mbuf_alloc_failed++; 1272 break; 1273 } 1274 1275 nb_hold++; 1276 rxe = &sw_ring[rx_id]; 1277 rx_id++; 1278 if (rx_id == rxq->nb_rx_desc) 1279 rx_id = 0; 1280 1281 /* Prefetch next mbuf while processing current one. */ 1282 rte_ngbe_prefetch(sw_ring[rx_id].mbuf); 1283 1284 /* 1285 * When next Rx descriptor is on a cache-line boundary, 1286 * prefetch the next 4 Rx descriptors and the next 8 pointers 1287 * to mbufs. 1288 */ 1289 if ((rx_id & 0x3) == 0) { 1290 rte_ngbe_prefetch(&rx_ring[rx_id]); 1291 rte_ngbe_prefetch(&sw_ring[rx_id]); 1292 } 1293 1294 rxm = rxe->mbuf; 1295 rxe->mbuf = nmb; 1296 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1297 NGBE_RXD_HDRADDR(rxdp, 0); 1298 NGBE_RXD_PKTADDR(rxdp, dma_addr); 1299 1300 /* 1301 * Initialize the returned mbuf. 1302 * 1) setup generic mbuf fields: 1303 * - number of segments, 1304 * - next segment, 1305 * - packet length, 1306 * - Rx port identifier. 1307 * 2) integrate hardware offload data, if any: 1308 * - RSS flag & hash, 1309 * - IP checksum flag, 1310 * - VLAN TCI, if any, 1311 * - error flags. 1312 */ 1313 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - 1314 rxq->crc_len); 1315 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1316 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1317 rxm->nb_segs = 1; 1318 rxm->next = NULL; 1319 rxm->pkt_len = pkt_len; 1320 rxm->data_len = pkt_len; 1321 rxm->port = rxq->port_id; 1322 1323 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); 1324 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ 1325 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); 1326 1327 pkt_flags = rx_desc_status_to_pkt_flags(staterr, 1328 rxq->vlan_flags); 1329 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1330 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1331 rxm->ol_flags = pkt_flags; 1332 rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1333 NGBE_PTID_MASK); 1334 1335 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1336 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); 1337 1338 /* 1339 * Store the mbuf address into the next entry of the array 1340 * of returned packets. 1341 */ 1342 rx_pkts[nb_rx++] = rxm; 1343 } 1344 rxq->rx_tail = rx_id; 1345 1346 /* 1347 * If the number of free Rx descriptors is greater than the Rx free 1348 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1349 * register. 1350 * Update the RDT with the value of the last processed Rx descriptor 1351 * minus 1, to guarantee that the RDT register is never equal to the 1352 * RDH register, which creates a "full" ring situation from the 1353 * hardware point of view... 1354 */ 1355 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1356 if (nb_hold > rxq->rx_free_thresh) { 1357 PMD_RX_LOG(DEBUG, 1358 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 1359 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1360 (uint16_t)rx_id, (uint16_t)nb_hold, 1361 (uint16_t)nb_rx); 1362 rx_id = (uint16_t)((rx_id == 0) ? 1363 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1364 ngbe_set32(rxq->rdt_reg_addr, rx_id); 1365 nb_hold = 0; 1366 } 1367 rxq->nb_rx_hold = nb_hold; 1368 return nb_rx; 1369 } 1370 1371 /** 1372 * ngbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1373 * 1374 * Fill the following info in the HEAD buffer of the Rx cluster: 1375 * - RX port identifier 1376 * - hardware offload data, if any: 1377 * - RSS flag & hash 1378 * - IP checksum flag 1379 * - VLAN TCI, if any 1380 * - error flags 1381 * @head HEAD of the packet cluster 1382 * @desc HW descriptor to get data from 1383 * @rxq Pointer to the Rx queue 1384 */ 1385 static inline void 1386 ngbe_fill_cluster_head_buf(struct rte_mbuf *head, struct ngbe_rx_desc *desc, 1387 struct ngbe_rx_queue *rxq, uint32_t staterr) 1388 { 1389 uint32_t pkt_info; 1390 uint64_t pkt_flags; 1391 1392 head->port = rxq->port_id; 1393 1394 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is 1395 * set in the pkt_flags field. 1396 */ 1397 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag); 1398 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0); 1399 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 1400 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1401 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1402 head->ol_flags = pkt_flags; 1403 head->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1404 NGBE_PTID_MASK); 1405 1406 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1407 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1); 1408 } 1409 1410 /** 1411 * ngbe_recv_pkts_sc - receive handler for scatter case. 1412 * 1413 * @rx_queue Rx queue handle 1414 * @rx_pkts table of received packets 1415 * @nb_pkts size of rx_pkts table 1416 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 1417 * 1418 * Returns the number of received packets/clusters (according to the "bulk 1419 * receive" interface). 1420 */ 1421 static inline uint16_t 1422 ngbe_recv_pkts_sc(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 1423 bool bulk_alloc) 1424 { 1425 struct ngbe_rx_queue *rxq = rx_queue; 1426 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1427 volatile struct ngbe_rx_desc *rx_ring = rxq->rx_ring; 1428 struct ngbe_rx_entry *sw_ring = rxq->sw_ring; 1429 struct ngbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 1430 uint16_t rx_id = rxq->rx_tail; 1431 uint16_t nb_rx = 0; 1432 uint16_t nb_hold = rxq->nb_rx_hold; 1433 uint16_t prev_id = rxq->rx_tail; 1434 1435 while (nb_rx < nb_pkts) { 1436 bool eop; 1437 struct ngbe_rx_entry *rxe; 1438 struct ngbe_scattered_rx_entry *sc_entry; 1439 struct ngbe_scattered_rx_entry *next_sc_entry = NULL; 1440 struct ngbe_rx_entry *next_rxe = NULL; 1441 struct rte_mbuf *first_seg; 1442 struct rte_mbuf *rxm; 1443 struct rte_mbuf *nmb = NULL; 1444 struct ngbe_rx_desc rxd; 1445 uint16_t data_len; 1446 uint16_t next_id; 1447 volatile struct ngbe_rx_desc *rxdp; 1448 uint32_t staterr; 1449 1450 next_desc: 1451 rxdp = &rx_ring[rx_id]; 1452 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); 1453 1454 if (!(staterr & NGBE_RXD_STAT_DD)) 1455 break; 1456 1457 rxd = *rxdp; 1458 1459 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1460 "staterr=0x%x data_len=%u", 1461 rxq->port_id, rxq->queue_id, rx_id, staterr, 1462 rte_le_to_cpu_16(rxd.qw1.hi.len)); 1463 1464 if (!bulk_alloc) { 1465 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1466 if (nmb == NULL) { 1467 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed " 1468 "port_id=%u queue_id=%u", 1469 rxq->port_id, rxq->queue_id); 1470 1471 dev->data->rx_mbuf_alloc_failed++; 1472 break; 1473 } 1474 } else if (nb_hold > rxq->rx_free_thresh) { 1475 uint16_t next_rdt = rxq->rx_free_trigger; 1476 1477 if (!ngbe_rx_alloc_bufs(rxq, false)) { 1478 rte_wmb(); 1479 ngbe_set32_relaxed(rxq->rdt_reg_addr, 1480 next_rdt); 1481 nb_hold -= rxq->rx_free_thresh; 1482 } else { 1483 PMD_RX_LOG(DEBUG, "Rx bulk alloc failed " 1484 "port_id=%u queue_id=%u", 1485 rxq->port_id, rxq->queue_id); 1486 1487 dev->data->rx_mbuf_alloc_failed++; 1488 break; 1489 } 1490 } 1491 1492 nb_hold++; 1493 rxe = &sw_ring[rx_id]; 1494 eop = staterr & NGBE_RXD_STAT_EOP; 1495 1496 next_id = rx_id + 1; 1497 if (next_id == rxq->nb_rx_desc) 1498 next_id = 0; 1499 1500 /* Prefetch next mbuf while processing current one. */ 1501 rte_ngbe_prefetch(sw_ring[next_id].mbuf); 1502 1503 /* 1504 * When next Rx descriptor is on a cache-line boundary, 1505 * prefetch the next 4 RX descriptors and the next 4 pointers 1506 * to mbufs. 1507 */ 1508 if ((next_id & 0x3) == 0) { 1509 rte_ngbe_prefetch(&rx_ring[next_id]); 1510 rte_ngbe_prefetch(&sw_ring[next_id]); 1511 } 1512 1513 rxm = rxe->mbuf; 1514 1515 if (!bulk_alloc) { 1516 __le64 dma = 1517 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1518 /* 1519 * Update Rx descriptor with the physical address of the 1520 * new data buffer of the new allocated mbuf. 1521 */ 1522 rxe->mbuf = nmb; 1523 1524 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1525 NGBE_RXD_HDRADDR(rxdp, 0); 1526 NGBE_RXD_PKTADDR(rxdp, dma); 1527 } else { 1528 rxe->mbuf = NULL; 1529 } 1530 1531 /* 1532 * Set data length & data buffer address of mbuf. 1533 */ 1534 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len); 1535 rxm->data_len = data_len; 1536 1537 if (!eop) { 1538 uint16_t nextp_id; 1539 1540 nextp_id = next_id; 1541 next_sc_entry = &sw_sc_ring[nextp_id]; 1542 next_rxe = &sw_ring[nextp_id]; 1543 rte_ngbe_prefetch(next_rxe); 1544 } 1545 1546 sc_entry = &sw_sc_ring[rx_id]; 1547 first_seg = sc_entry->fbuf; 1548 sc_entry->fbuf = NULL; 1549 1550 /* 1551 * If this is the first buffer of the received packet, 1552 * set the pointer to the first mbuf of the packet and 1553 * initialize its context. 1554 * Otherwise, update the total length and the number of segments 1555 * of the current scattered packet, and update the pointer to 1556 * the last mbuf of the current packet. 1557 */ 1558 if (first_seg == NULL) { 1559 first_seg = rxm; 1560 first_seg->pkt_len = data_len; 1561 first_seg->nb_segs = 1; 1562 } else { 1563 first_seg->pkt_len += data_len; 1564 first_seg->nb_segs++; 1565 } 1566 1567 prev_id = rx_id; 1568 rx_id = next_id; 1569 1570 /* 1571 * If this is not the last buffer of the received packet, update 1572 * the pointer to the first mbuf at the NEXTP entry in the 1573 * sw_sc_ring and continue to parse the Rx ring. 1574 */ 1575 if (!eop && next_rxe) { 1576 rxm->next = next_rxe->mbuf; 1577 next_sc_entry->fbuf = first_seg; 1578 goto next_desc; 1579 } 1580 1581 /* Initialize the first mbuf of the returned packet */ 1582 ngbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 1583 1584 /* Deal with the case, when HW CRC srip is disabled. */ 1585 first_seg->pkt_len -= rxq->crc_len; 1586 if (unlikely(rxm->data_len <= rxq->crc_len)) { 1587 struct rte_mbuf *lp; 1588 1589 for (lp = first_seg; lp->next != rxm; lp = lp->next) 1590 ; 1591 1592 first_seg->nb_segs--; 1593 lp->data_len -= rxq->crc_len - rxm->data_len; 1594 lp->next = NULL; 1595 rte_pktmbuf_free_seg(rxm); 1596 } else { 1597 rxm->data_len -= rxq->crc_len; 1598 } 1599 1600 /* Prefetch data of first segment, if configured to do so. */ 1601 rte_packet_prefetch((char *)first_seg->buf_addr + 1602 first_seg->data_off); 1603 1604 /* 1605 * Store the mbuf address into the next entry of the array 1606 * of returned packets. 1607 */ 1608 rx_pkts[nb_rx++] = first_seg; 1609 } 1610 1611 /* 1612 * Record index of the next Rx descriptor to probe. 1613 */ 1614 rxq->rx_tail = rx_id; 1615 1616 /* 1617 * If the number of free Rx descriptors is greater than the Rx free 1618 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1619 * register. 1620 * Update the RDT with the value of the last processed Rx descriptor 1621 * minus 1, to guarantee that the RDT register is never equal to the 1622 * RDH register, which creates a "full" ring situation from the 1623 * hardware point of view... 1624 */ 1625 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 1626 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1627 "nb_hold=%u nb_rx=%u", 1628 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 1629 1630 rte_wmb(); 1631 ngbe_set32_relaxed(rxq->rdt_reg_addr, prev_id); 1632 nb_hold = 0; 1633 } 1634 1635 rxq->nb_rx_hold = nb_hold; 1636 return nb_rx; 1637 } 1638 1639 uint16_t 1640 ngbe_recv_pkts_sc_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1641 uint16_t nb_pkts) 1642 { 1643 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, false); 1644 } 1645 1646 uint16_t 1647 ngbe_recv_pkts_sc_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1648 uint16_t nb_pkts) 1649 { 1650 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, true); 1651 } 1652 1653 /********************************************************************* 1654 * 1655 * Queue management functions 1656 * 1657 **********************************************************************/ 1658 1659 static void 1660 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq) 1661 { 1662 unsigned int i; 1663 1664 if (txq->sw_ring != NULL) { 1665 for (i = 0; i < txq->nb_tx_desc; i++) { 1666 if (txq->sw_ring[i].mbuf != NULL) { 1667 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 1668 txq->sw_ring[i].mbuf = NULL; 1669 } 1670 } 1671 } 1672 } 1673 1674 static int 1675 ngbe_tx_done_cleanup_full(struct ngbe_tx_queue *txq, uint32_t free_cnt) 1676 { 1677 struct ngbe_tx_entry *swr_ring = txq->sw_ring; 1678 uint16_t i, tx_last, tx_id; 1679 uint16_t nb_tx_free_last; 1680 uint16_t nb_tx_to_clean; 1681 uint32_t pkt_cnt; 1682 1683 /* Start free mbuf from the next of tx_tail */ 1684 tx_last = txq->tx_tail; 1685 tx_id = swr_ring[tx_last].next_id; 1686 1687 if (txq->nb_tx_free == 0 && ngbe_xmit_cleanup(txq)) 1688 return 0; 1689 1690 nb_tx_to_clean = txq->nb_tx_free; 1691 nb_tx_free_last = txq->nb_tx_free; 1692 if (!free_cnt) 1693 free_cnt = txq->nb_tx_desc; 1694 1695 /* Loop through swr_ring to count the amount of 1696 * freeable mubfs and packets. 1697 */ 1698 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 1699 for (i = 0; i < nb_tx_to_clean && 1700 pkt_cnt < free_cnt && 1701 tx_id != tx_last; i++) { 1702 if (swr_ring[tx_id].mbuf != NULL) { 1703 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 1704 swr_ring[tx_id].mbuf = NULL; 1705 1706 /* 1707 * last segment in the packet, 1708 * increment packet count 1709 */ 1710 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 1711 } 1712 1713 tx_id = swr_ring[tx_id].next_id; 1714 } 1715 1716 if (pkt_cnt < free_cnt) { 1717 if (ngbe_xmit_cleanup(txq)) 1718 break; 1719 1720 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 1721 nb_tx_free_last = txq->nb_tx_free; 1722 } 1723 } 1724 1725 return (int)pkt_cnt; 1726 } 1727 1728 static int 1729 ngbe_tx_done_cleanup_simple(struct ngbe_tx_queue *txq, 1730 uint32_t free_cnt) 1731 { 1732 int i, n, cnt; 1733 1734 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 1735 free_cnt = txq->nb_tx_desc; 1736 1737 cnt = free_cnt - free_cnt % txq->tx_free_thresh; 1738 1739 for (i = 0; i < cnt; i += n) { 1740 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh) 1741 break; 1742 1743 n = ngbe_tx_free_bufs(txq); 1744 1745 if (n == 0) 1746 break; 1747 } 1748 1749 return i; 1750 } 1751 1752 int 1753 ngbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 1754 { 1755 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 1756 if (txq->offloads == 0 && 1757 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) 1758 return ngbe_tx_done_cleanup_simple(txq, free_cnt); 1759 1760 return ngbe_tx_done_cleanup_full(txq, free_cnt); 1761 } 1762 1763 static void 1764 ngbe_tx_free_swring(struct ngbe_tx_queue *txq) 1765 { 1766 if (txq != NULL) 1767 rte_free(txq->sw_ring); 1768 } 1769 1770 static void 1771 ngbe_tx_queue_release(struct ngbe_tx_queue *txq) 1772 { 1773 if (txq != NULL) { 1774 if (txq->ops != NULL) { 1775 txq->ops->release_mbufs(txq); 1776 txq->ops->free_swring(txq); 1777 } 1778 rte_free(txq); 1779 } 1780 } 1781 1782 void 1783 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1784 { 1785 ngbe_tx_queue_release(dev->data->tx_queues[qid]); 1786 } 1787 1788 /* (Re)set dynamic ngbe_tx_queue fields to defaults */ 1789 static void 1790 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq) 1791 { 1792 static const struct ngbe_tx_desc zeroed_desc = {0}; 1793 struct ngbe_tx_entry *txe = txq->sw_ring; 1794 uint16_t prev, i; 1795 1796 /* Zero out HW ring memory */ 1797 for (i = 0; i < txq->nb_tx_desc; i++) 1798 txq->tx_ring[i] = zeroed_desc; 1799 1800 /* Initialize SW ring entries */ 1801 prev = (uint16_t)(txq->nb_tx_desc - 1); 1802 for (i = 0; i < txq->nb_tx_desc; i++) { 1803 /* the ring can also be modified by hardware */ 1804 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i]; 1805 1806 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD); 1807 txe[i].mbuf = NULL; 1808 txe[i].last_id = i; 1809 txe[prev].next_id = i; 1810 prev = i; 1811 } 1812 1813 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 1814 txq->tx_tail = 0; 1815 1816 /* 1817 * Always allow 1 descriptor to be un-allocated to avoid 1818 * a H/W race condition 1819 */ 1820 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 1821 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 1822 txq->ctx_curr = 0; 1823 memset((void *)&txq->ctx_cache, 0, 1824 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info)); 1825 } 1826 1827 static const struct ngbe_txq_ops def_txq_ops = { 1828 .release_mbufs = ngbe_tx_queue_release_mbufs, 1829 .free_swring = ngbe_tx_free_swring, 1830 .reset = ngbe_reset_tx_queue, 1831 }; 1832 1833 /* Takes an ethdev and a queue and sets up the tx function to be used based on 1834 * the queue parameters. Used in tx_queue_setup by primary process and then 1835 * in dev_init by secondary process when attaching to an existing ethdev. 1836 */ 1837 void 1838 ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq) 1839 { 1840 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 1841 if (txq->offloads == 0 && 1842 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) { 1843 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 1844 dev->tx_pkt_burst = ngbe_xmit_pkts_simple; 1845 dev->tx_pkt_prepare = NULL; 1846 } else { 1847 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 1848 PMD_INIT_LOG(DEBUG, 1849 " - offloads = 0x%" PRIx64, 1850 txq->offloads); 1851 PMD_INIT_LOG(DEBUG, 1852 " - tx_free_thresh = %lu [RTE_PMD_NGBE_TX_MAX_BURST=%lu]", 1853 (unsigned long)txq->tx_free_thresh, 1854 (unsigned long)RTE_PMD_NGBE_TX_MAX_BURST); 1855 dev->tx_pkt_burst = ngbe_xmit_pkts; 1856 dev->tx_pkt_prepare = ngbe_prep_pkts; 1857 } 1858 } 1859 1860 static const struct { 1861 eth_tx_burst_t pkt_burst; 1862 const char *info; 1863 } ngbe_tx_burst_infos[] = { 1864 { ngbe_xmit_pkts_simple, "Scalar Simple"}, 1865 { ngbe_xmit_pkts, "Scalar"}, 1866 }; 1867 1868 int 1869 ngbe_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 1870 struct rte_eth_burst_mode *mode) 1871 { 1872 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 1873 int ret = -EINVAL; 1874 unsigned int i; 1875 1876 for (i = 0; i < RTE_DIM(ngbe_tx_burst_infos); ++i) { 1877 if (pkt_burst == ngbe_tx_burst_infos[i].pkt_burst) { 1878 snprintf(mode->info, sizeof(mode->info), "%s", 1879 ngbe_tx_burst_infos[i].info); 1880 ret = 0; 1881 break; 1882 } 1883 } 1884 1885 return ret; 1886 } 1887 1888 uint64_t 1889 ngbe_get_tx_port_offloads(struct rte_eth_dev *dev) 1890 { 1891 uint64_t tx_offload_capa; 1892 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1893 1894 tx_offload_capa = 1895 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 1896 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 1897 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 1898 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 1899 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 1900 RTE_ETH_TX_OFFLOAD_TCP_TSO | 1901 RTE_ETH_TX_OFFLOAD_UDP_TSO | 1902 RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 1903 1904 if (hw->is_pf) 1905 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT; 1906 1907 return tx_offload_capa; 1908 } 1909 1910 int 1911 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 1912 uint16_t queue_idx, 1913 uint16_t nb_desc, 1914 unsigned int socket_id, 1915 const struct rte_eth_txconf *tx_conf) 1916 { 1917 const struct rte_memzone *tz; 1918 struct ngbe_tx_queue *txq; 1919 struct ngbe_hw *hw; 1920 uint16_t tx_free_thresh; 1921 uint64_t offloads; 1922 1923 PMD_INIT_FUNC_TRACE(); 1924 hw = ngbe_dev_hw(dev); 1925 1926 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1927 1928 /* 1929 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh 1930 * descriptors are used or if the number of descriptors required 1931 * to transmit a packet is greater than the number of free Tx 1932 * descriptors. 1933 * One descriptor in the Tx ring is used as a sentinel to avoid a 1934 * H/W race condition, hence the maximum threshold constraints. 1935 * When set to zero use default values. 1936 */ 1937 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 1938 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 1939 if (tx_free_thresh >= (nb_desc - 3)) { 1940 PMD_INIT_LOG(ERR, 1941 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)", 1942 (unsigned int)tx_free_thresh, 1943 (int)dev->data->port_id, (int)queue_idx); 1944 return -(EINVAL); 1945 } 1946 1947 if (nb_desc % tx_free_thresh != 0) { 1948 PMD_INIT_LOG(ERR, 1949 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)", 1950 (unsigned int)tx_free_thresh, 1951 (int)dev->data->port_id, (int)queue_idx); 1952 return -(EINVAL); 1953 } 1954 1955 /* Free memory prior to re-allocation if needed... */ 1956 if (dev->data->tx_queues[queue_idx] != NULL) { 1957 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 1958 dev->data->tx_queues[queue_idx] = NULL; 1959 } 1960 1961 /* First allocate the Tx queue data structure */ 1962 txq = rte_zmalloc_socket("ethdev Tx queue", 1963 sizeof(struct ngbe_tx_queue), 1964 RTE_CACHE_LINE_SIZE, socket_id); 1965 if (txq == NULL) 1966 return -ENOMEM; 1967 1968 /* 1969 * Allocate Tx ring hardware descriptors. A memzone large enough to 1970 * handle the maximum ring size is allocated in order to allow for 1971 * resizing in later calls to the queue setup function. 1972 */ 1973 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 1974 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX, 1975 NGBE_ALIGN, socket_id); 1976 if (tz == NULL) { 1977 ngbe_tx_queue_release(txq); 1978 return -ENOMEM; 1979 } 1980 1981 txq->nb_tx_desc = nb_desc; 1982 txq->tx_free_thresh = tx_free_thresh; 1983 txq->pthresh = tx_conf->tx_thresh.pthresh; 1984 txq->hthresh = tx_conf->tx_thresh.hthresh; 1985 txq->wthresh = tx_conf->tx_thresh.wthresh; 1986 txq->queue_id = queue_idx; 1987 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 1988 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 1989 txq->port_id = dev->data->port_id; 1990 txq->offloads = offloads; 1991 txq->ops = &def_txq_ops; 1992 txq->tx_deferred_start = tx_conf->tx_deferred_start; 1993 1994 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx)); 1995 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx)); 1996 1997 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 1998 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz); 1999 2000 /* Allocate software ring */ 2001 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2002 sizeof(struct ngbe_tx_entry) * nb_desc, 2003 RTE_CACHE_LINE_SIZE, socket_id); 2004 if (txq->sw_ring == NULL) { 2005 ngbe_tx_queue_release(txq); 2006 return -ENOMEM; 2007 } 2008 PMD_INIT_LOG(DEBUG, 2009 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2010 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2011 2012 /* set up scalar Tx function as appropriate */ 2013 ngbe_set_tx_function(dev, txq); 2014 2015 txq->ops->reset(txq); 2016 2017 dev->data->tx_queues[queue_idx] = txq; 2018 2019 return 0; 2020 } 2021 2022 /** 2023 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster 2024 * 2025 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2026 * in the sw_sc_ring is not set to NULL but rather points to the next 2027 * mbuf of this RSC aggregation (that has not been completed yet and still 2028 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2029 * will just free first "nb_segs" segments of the cluster explicitly by calling 2030 * an rte_pktmbuf_free_seg(). 2031 * 2032 * @m scattered cluster head 2033 */ 2034 static void 2035 ngbe_free_sc_cluster(struct rte_mbuf *m) 2036 { 2037 uint16_t i, nb_segs = m->nb_segs; 2038 struct rte_mbuf *next_seg; 2039 2040 for (i = 0; i < nb_segs; i++) { 2041 next_seg = m->next; 2042 rte_pktmbuf_free_seg(m); 2043 m = next_seg; 2044 } 2045 } 2046 2047 static void 2048 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq) 2049 { 2050 unsigned int i; 2051 2052 if (rxq->sw_ring != NULL) { 2053 for (i = 0; i < rxq->nb_rx_desc; i++) { 2054 if (rxq->sw_ring[i].mbuf != NULL) { 2055 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2056 rxq->sw_ring[i].mbuf = NULL; 2057 } 2058 } 2059 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2060 struct rte_mbuf *mb; 2061 2062 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2063 rte_pktmbuf_free_seg(mb); 2064 } 2065 rxq->rx_nb_avail = 0; 2066 } 2067 2068 if (rxq->sw_sc_ring != NULL) 2069 for (i = 0; i < rxq->nb_rx_desc; i++) 2070 if (rxq->sw_sc_ring[i].fbuf != NULL) { 2071 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2072 rxq->sw_sc_ring[i].fbuf = NULL; 2073 } 2074 } 2075 2076 static void 2077 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq) 2078 { 2079 if (rxq != NULL) { 2080 ngbe_rx_queue_release_mbufs(rxq); 2081 rte_free(rxq->sw_ring); 2082 rte_free(rxq->sw_sc_ring); 2083 rte_free(rxq); 2084 } 2085 } 2086 2087 void 2088 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2089 { 2090 ngbe_rx_queue_release(dev->data->rx_queues[qid]); 2091 } 2092 2093 /* 2094 * Check if Rx Burst Bulk Alloc function can be used. 2095 * Return 2096 * 0: the preconditions are satisfied and the bulk allocation function 2097 * can be used. 2098 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2099 * function must be used. 2100 */ 2101 static inline int 2102 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq) 2103 { 2104 int ret = 0; 2105 2106 /* 2107 * Make sure the following pre-conditions are satisfied: 2108 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST 2109 * rxq->rx_free_thresh < rxq->nb_rx_desc 2110 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2111 * Scattered packets are not supported. This should be checked 2112 * outside of this function. 2113 */ 2114 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) { 2115 PMD_INIT_LOG(DEBUG, 2116 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d", 2117 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST); 2118 ret = -EINVAL; 2119 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) { 2120 PMD_INIT_LOG(DEBUG, 2121 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d", 2122 rxq->rx_free_thresh, rxq->nb_rx_desc); 2123 ret = -EINVAL; 2124 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) { 2125 PMD_INIT_LOG(DEBUG, 2126 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d", 2127 rxq->nb_rx_desc, rxq->rx_free_thresh); 2128 ret = -EINVAL; 2129 } 2130 2131 return ret; 2132 } 2133 2134 /* Reset dynamic ngbe_rx_queue fields back to defaults */ 2135 static void 2136 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq) 2137 { 2138 static const struct ngbe_rx_desc zeroed_desc = { 2139 {{0}, {0} }, {{0}, {0} } }; 2140 unsigned int i; 2141 uint16_t len = rxq->nb_rx_desc; 2142 2143 /* 2144 * By default, the Rx queue setup function allocates enough memory for 2145 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 2146 * extra memory at the end of the descriptor ring to be zero'd out. 2147 */ 2148 if (adapter->rx_bulk_alloc_allowed) 2149 /* zero out extra memory */ 2150 len += RTE_PMD_NGBE_RX_MAX_BURST; 2151 2152 /* 2153 * Zero out HW ring memory. Zero out extra memory at the end of 2154 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2155 * reads extra memory as zeros. 2156 */ 2157 for (i = 0; i < len; i++) 2158 rxq->rx_ring[i] = zeroed_desc; 2159 2160 /* 2161 * initialize extra software ring entries. Space for these extra 2162 * entries is always allocated 2163 */ 2164 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2165 for (i = rxq->nb_rx_desc; i < len; ++i) 2166 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2167 2168 rxq->rx_nb_avail = 0; 2169 rxq->rx_next_avail = 0; 2170 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2171 rxq->rx_tail = 0; 2172 rxq->nb_rx_hold = 0; 2173 rxq->pkt_first_seg = NULL; 2174 rxq->pkt_last_seg = NULL; 2175 } 2176 2177 uint64_t 2178 ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused) 2179 { 2180 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2181 } 2182 2183 uint64_t 2184 ngbe_get_rx_port_offloads(struct rte_eth_dev *dev) 2185 { 2186 uint64_t offloads; 2187 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2188 2189 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 2190 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 2191 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 2192 RTE_ETH_RX_OFFLOAD_KEEP_CRC | 2193 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 2194 RTE_ETH_RX_OFFLOAD_SCATTER; 2195 2196 if (hw->is_pf) 2197 offloads |= (RTE_ETH_RX_OFFLOAD_QINQ_STRIP | 2198 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND); 2199 2200 return offloads; 2201 } 2202 2203 int 2204 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 2205 uint16_t queue_idx, 2206 uint16_t nb_desc, 2207 unsigned int socket_id, 2208 const struct rte_eth_rxconf *rx_conf, 2209 struct rte_mempool *mp) 2210 { 2211 const struct rte_memzone *rz; 2212 struct ngbe_rx_queue *rxq; 2213 struct ngbe_hw *hw; 2214 uint16_t len; 2215 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2216 uint64_t offloads; 2217 2218 PMD_INIT_FUNC_TRACE(); 2219 hw = ngbe_dev_hw(dev); 2220 2221 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2222 2223 /* Free memory prior to re-allocation if needed... */ 2224 if (dev->data->rx_queues[queue_idx] != NULL) { 2225 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 2226 dev->data->rx_queues[queue_idx] = NULL; 2227 } 2228 2229 /* First allocate the Rx queue data structure */ 2230 rxq = rte_zmalloc_socket("ethdev RX queue", 2231 sizeof(struct ngbe_rx_queue), 2232 RTE_CACHE_LINE_SIZE, socket_id); 2233 if (rxq == NULL) 2234 return -ENOMEM; 2235 rxq->mb_pool = mp; 2236 rxq->nb_rx_desc = nb_desc; 2237 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2238 rxq->queue_id = queue_idx; 2239 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2240 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2241 rxq->port_id = dev->data->port_id; 2242 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2243 rxq->crc_len = RTE_ETHER_CRC_LEN; 2244 else 2245 rxq->crc_len = 0; 2246 rxq->drop_en = rx_conf->rx_drop_en; 2247 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2248 rxq->offloads = offloads; 2249 2250 /* 2251 * Allocate Rx ring hardware descriptors. A memzone large enough to 2252 * handle the maximum ring size is allocated in order to allow for 2253 * resizing in later calls to the queue setup function. 2254 */ 2255 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2256 RX_RING_SZ, NGBE_ALIGN, socket_id); 2257 if (rz == NULL) { 2258 ngbe_rx_queue_release(rxq); 2259 return -ENOMEM; 2260 } 2261 2262 /* 2263 * Zero init all the descriptors in the ring. 2264 */ 2265 memset(rz->addr, 0, RX_RING_SZ); 2266 2267 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx)); 2268 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx)); 2269 2270 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 2271 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz); 2272 2273 /* 2274 * Certain constraints must be met in order to use the bulk buffer 2275 * allocation Rx burst function. If any of Rx queues doesn't meet them 2276 * the feature should be disabled for the whole port. 2277 */ 2278 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 2279 PMD_INIT_LOG(DEBUG, 2280 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]", 2281 rxq->queue_id, rxq->port_id); 2282 adapter->rx_bulk_alloc_allowed = false; 2283 } 2284 2285 /* 2286 * Allocate software ring. Allow for space at the end of the 2287 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 2288 * function does not access an invalid memory region. 2289 */ 2290 len = nb_desc; 2291 if (adapter->rx_bulk_alloc_allowed) 2292 len += RTE_PMD_NGBE_RX_MAX_BURST; 2293 2294 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 2295 sizeof(struct ngbe_rx_entry) * len, 2296 RTE_CACHE_LINE_SIZE, socket_id); 2297 if (rxq->sw_ring == NULL) { 2298 ngbe_rx_queue_release(rxq); 2299 return -ENOMEM; 2300 } 2301 2302 /* 2303 * Always allocate even if it's not going to be needed in order to 2304 * simplify the code. 2305 * 2306 * This ring is used in Scattered Rx cases and Scattered Rx may 2307 * be requested in ngbe_dev_rx_init(), which is called later from 2308 * dev_start() flow. 2309 */ 2310 rxq->sw_sc_ring = 2311 rte_zmalloc_socket("rxq->sw_sc_ring", 2312 sizeof(struct ngbe_scattered_rx_entry) * len, 2313 RTE_CACHE_LINE_SIZE, socket_id); 2314 if (rxq->sw_sc_ring == NULL) { 2315 ngbe_rx_queue_release(rxq); 2316 return -ENOMEM; 2317 } 2318 2319 PMD_INIT_LOG(DEBUG, 2320 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2321 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 2322 rxq->rx_ring_phys_addr); 2323 2324 dev->data->rx_queues[queue_idx] = rxq; 2325 2326 ngbe_reset_rx_queue(adapter, rxq); 2327 2328 return 0; 2329 } 2330 2331 uint32_t 2332 ngbe_dev_rx_queue_count(void *rx_queue) 2333 { 2334 #define NGBE_RXQ_SCAN_INTERVAL 4 2335 volatile struct ngbe_rx_desc *rxdp; 2336 struct ngbe_rx_queue *rxq = rx_queue; 2337 uint32_t desc = 0; 2338 2339 rxdp = &rxq->rx_ring[rxq->rx_tail]; 2340 2341 while ((desc < rxq->nb_rx_desc) && 2342 (rxdp->qw1.lo.status & 2343 rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) { 2344 desc += NGBE_RXQ_SCAN_INTERVAL; 2345 rxdp += NGBE_RXQ_SCAN_INTERVAL; 2346 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2347 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2348 desc - rxq->nb_rx_desc]); 2349 } 2350 2351 return desc; 2352 } 2353 2354 int 2355 ngbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2356 { 2357 struct ngbe_rx_queue *rxq = rx_queue; 2358 volatile uint32_t *status; 2359 uint32_t nb_hold, desc; 2360 2361 if (unlikely(offset >= rxq->nb_rx_desc)) 2362 return -EINVAL; 2363 2364 nb_hold = rxq->nb_rx_hold; 2365 if (offset >= rxq->nb_rx_desc - nb_hold) 2366 return RTE_ETH_RX_DESC_UNAVAIL; 2367 2368 desc = rxq->rx_tail + offset; 2369 if (desc >= rxq->nb_rx_desc) 2370 desc -= rxq->nb_rx_desc; 2371 2372 status = &rxq->rx_ring[desc].qw1.lo.status; 2373 if (*status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)) 2374 return RTE_ETH_RX_DESC_DONE; 2375 2376 return RTE_ETH_RX_DESC_AVAIL; 2377 } 2378 2379 int 2380 ngbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2381 { 2382 struct ngbe_tx_queue *txq = tx_queue; 2383 volatile uint32_t *status; 2384 uint32_t desc; 2385 2386 if (unlikely(offset >= txq->nb_tx_desc)) 2387 return -EINVAL; 2388 2389 desc = txq->tx_tail + offset; 2390 if (desc >= txq->nb_tx_desc) { 2391 desc -= txq->nb_tx_desc; 2392 if (desc >= txq->nb_tx_desc) 2393 desc -= txq->nb_tx_desc; 2394 } 2395 2396 status = &txq->tx_ring[desc].dw3; 2397 if (*status & rte_cpu_to_le_32(NGBE_TXD_DD)) 2398 return RTE_ETH_TX_DESC_DONE; 2399 2400 return RTE_ETH_TX_DESC_FULL; 2401 } 2402 2403 void 2404 ngbe_dev_clear_queues(struct rte_eth_dev *dev) 2405 { 2406 unsigned int i; 2407 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2408 2409 PMD_INIT_FUNC_TRACE(); 2410 2411 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2412 struct ngbe_tx_queue *txq = dev->data->tx_queues[i]; 2413 2414 if (txq != NULL) { 2415 txq->ops->release_mbufs(txq); 2416 txq->ops->reset(txq); 2417 } 2418 } 2419 2420 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2421 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i]; 2422 2423 if (rxq != NULL) { 2424 ngbe_rx_queue_release_mbufs(rxq); 2425 ngbe_reset_rx_queue(adapter, rxq); 2426 } 2427 } 2428 } 2429 2430 void 2431 ngbe_dev_free_queues(struct rte_eth_dev *dev) 2432 { 2433 unsigned int i; 2434 2435 PMD_INIT_FUNC_TRACE(); 2436 2437 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2438 ngbe_dev_rx_queue_release(dev, i); 2439 dev->data->rx_queues[i] = NULL; 2440 } 2441 dev->data->nb_rx_queues = 0; 2442 2443 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2444 ngbe_dev_tx_queue_release(dev, i); 2445 dev->data->tx_queues[i] = NULL; 2446 } 2447 dev->data->nb_tx_queues = 0; 2448 } 2449 2450 /** 2451 * Receive Side Scaling (RSS) 2452 * 2453 * Principles: 2454 * The source and destination IP addresses of the IP header and the source 2455 * and destination ports of TCP/UDP headers, if any, of received packets are 2456 * hashed against a configurable random key to compute a 32-bit RSS hash result. 2457 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 2458 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 2459 * RSS output index which is used as the Rx queue index where to store the 2460 * received packets. 2461 * The following output is supplied in the Rx write-back descriptor: 2462 * - 32-bit result of the Microsoft RSS hash function, 2463 * - 4-bit RSS type field. 2464 */ 2465 2466 /* 2467 * Used as the default key. 2468 */ 2469 static uint8_t rss_intel_key[40] = { 2470 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 2471 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 2472 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 2473 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 2474 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 2475 }; 2476 2477 static void 2478 ngbe_rss_disable(struct rte_eth_dev *dev) 2479 { 2480 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2481 2482 wr32m(hw, NGBE_RACTL, NGBE_RACTL_RSSENA, 0); 2483 } 2484 2485 int 2486 ngbe_dev_rss_hash_update(struct rte_eth_dev *dev, 2487 struct rte_eth_rss_conf *rss_conf) 2488 { 2489 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2490 uint8_t *hash_key; 2491 uint32_t mrqc; 2492 uint32_t rss_key; 2493 uint64_t rss_hf; 2494 uint16_t i; 2495 2496 if (!hw->is_pf) { 2497 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 2498 "NIC."); 2499 return -ENOTSUP; 2500 } 2501 2502 hash_key = rss_conf->rss_key; 2503 if (hash_key) { 2504 /* Fill in RSS hash key */ 2505 for (i = 0; i < 10; i++) { 2506 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); 2507 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); 2508 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); 2509 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); 2510 wr32a(hw, NGBE_REG_RSSKEY, i, rss_key); 2511 } 2512 } 2513 2514 /* Set configured hashing protocols */ 2515 rss_hf = rss_conf->rss_hf & NGBE_RSS_OFFLOAD_ALL; 2516 2517 mrqc = rd32(hw, NGBE_RACTL); 2518 mrqc &= ~NGBE_RACTL_RSSMASK; 2519 if (rss_hf & RTE_ETH_RSS_IPV4) 2520 mrqc |= NGBE_RACTL_RSSIPV4; 2521 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 2522 mrqc |= NGBE_RACTL_RSSIPV4TCP; 2523 if (rss_hf & RTE_ETH_RSS_IPV6 || 2524 rss_hf & RTE_ETH_RSS_IPV6_EX) 2525 mrqc |= NGBE_RACTL_RSSIPV6; 2526 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP || 2527 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 2528 mrqc |= NGBE_RACTL_RSSIPV6TCP; 2529 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 2530 mrqc |= NGBE_RACTL_RSSIPV4UDP; 2531 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP || 2532 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 2533 mrqc |= NGBE_RACTL_RSSIPV6UDP; 2534 2535 if (rss_hf) 2536 mrqc |= NGBE_RACTL_RSSENA; 2537 else 2538 mrqc &= ~NGBE_RACTL_RSSENA; 2539 2540 wr32(hw, NGBE_RACTL, mrqc); 2541 2542 return 0; 2543 } 2544 2545 int 2546 ngbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2547 struct rte_eth_rss_conf *rss_conf) 2548 { 2549 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2550 uint8_t *hash_key; 2551 uint32_t mrqc; 2552 uint32_t rss_key; 2553 uint64_t rss_hf; 2554 uint16_t i; 2555 2556 hash_key = rss_conf->rss_key; 2557 if (hash_key) { 2558 /* Return RSS hash key */ 2559 for (i = 0; i < 10; i++) { 2560 rss_key = rd32a(hw, NGBE_REG_RSSKEY, i); 2561 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); 2562 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); 2563 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); 2564 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); 2565 } 2566 } 2567 2568 rss_hf = 0; 2569 2570 mrqc = rd32(hw, NGBE_RACTL); 2571 if (mrqc & NGBE_RACTL_RSSIPV4) 2572 rss_hf |= RTE_ETH_RSS_IPV4; 2573 if (mrqc & NGBE_RACTL_RSSIPV4TCP) 2574 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 2575 if (mrqc & NGBE_RACTL_RSSIPV6) 2576 rss_hf |= RTE_ETH_RSS_IPV6 | 2577 RTE_ETH_RSS_IPV6_EX; 2578 if (mrqc & NGBE_RACTL_RSSIPV6TCP) 2579 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | 2580 RTE_ETH_RSS_IPV6_TCP_EX; 2581 if (mrqc & NGBE_RACTL_RSSIPV4UDP) 2582 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 2583 if (mrqc & NGBE_RACTL_RSSIPV6UDP) 2584 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | 2585 RTE_ETH_RSS_IPV6_UDP_EX; 2586 if (!(mrqc & NGBE_RACTL_RSSENA)) 2587 rss_hf = 0; 2588 2589 rss_hf &= NGBE_RSS_OFFLOAD_ALL; 2590 2591 rss_conf->rss_hf = rss_hf; 2592 return 0; 2593 } 2594 2595 static void 2596 ngbe_rss_configure(struct rte_eth_dev *dev) 2597 { 2598 struct rte_eth_rss_conf rss_conf; 2599 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2600 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2601 uint32_t reta; 2602 uint16_t i; 2603 uint16_t j; 2604 2605 PMD_INIT_FUNC_TRACE(); 2606 2607 /* 2608 * Fill in redirection table 2609 * The byte-swap is needed because NIC registers are in 2610 * little-endian order. 2611 */ 2612 if (adapter->rss_reta_updated == 0) { 2613 reta = 0; 2614 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { 2615 if (j == dev->data->nb_rx_queues) 2616 j = 0; 2617 reta = (reta >> 8) | LS32(j, 24, 0xFF); 2618 if ((i & 3) == 3) 2619 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta); 2620 } 2621 } 2622 /* 2623 * Configure the RSS key and the RSS protocols used to compute 2624 * the RSS hash of input packets. 2625 */ 2626 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 2627 if (rss_conf.rss_key == NULL) 2628 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 2629 ngbe_dev_rss_hash_update(dev, &rss_conf); 2630 } 2631 2632 void ngbe_configure_port(struct rte_eth_dev *dev) 2633 { 2634 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2635 int i = 0; 2636 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, 2637 0x9100, 0x9200, 2638 0x0000, 0x0000, 2639 0x0000, 0x0000}; 2640 2641 PMD_INIT_FUNC_TRACE(); 2642 2643 /* default outer vlan tpid */ 2644 wr32(hw, NGBE_EXTAG, 2645 NGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | 2646 NGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); 2647 2648 /* default inner vlan tpid */ 2649 wr32m(hw, NGBE_VLANCTL, 2650 NGBE_VLANCTL_TPID_MASK, 2651 NGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); 2652 wr32m(hw, NGBE_DMATXCTRL, 2653 NGBE_DMATXCTRL_TPID_MASK, 2654 NGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); 2655 2656 /* default vlan tpid filters */ 2657 for (i = 0; i < 8; i++) { 2658 wr32m(hw, NGBE_TAGTPID(i / 2), 2659 (i % 2 ? NGBE_TAGTPID_MSB_MASK 2660 : NGBE_TAGTPID_LSB_MASK), 2661 (i % 2 ? NGBE_TAGTPID_MSB(tpids[i]) 2662 : NGBE_TAGTPID_LSB(tpids[i]))); 2663 } 2664 } 2665 2666 static int 2667 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq) 2668 { 2669 struct ngbe_rx_entry *rxe = rxq->sw_ring; 2670 uint64_t dma_addr; 2671 unsigned int i; 2672 2673 /* Initialize software ring entries */ 2674 for (i = 0; i < rxq->nb_rx_desc; i++) { 2675 /* the ring can also be modified by hardware */ 2676 volatile struct ngbe_rx_desc *rxd; 2677 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 2678 2679 if (mbuf == NULL) { 2680 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u", 2681 (unsigned int)rxq->queue_id, 2682 (unsigned int)rxq->port_id); 2683 return -ENOMEM; 2684 } 2685 2686 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2687 mbuf->port = rxq->port_id; 2688 2689 dma_addr = 2690 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 2691 rxd = &rxq->rx_ring[i]; 2692 NGBE_RXD_HDRADDR(rxd, 0); 2693 NGBE_RXD_PKTADDR(rxd, dma_addr); 2694 rxe[i].mbuf = mbuf; 2695 } 2696 2697 return 0; 2698 } 2699 2700 static int 2701 ngbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 2702 { 2703 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 2704 switch (dev->data->dev_conf.rxmode.mq_mode) { 2705 case RTE_ETH_MQ_RX_RSS: 2706 ngbe_rss_configure(dev); 2707 break; 2708 2709 case RTE_ETH_MQ_RX_NONE: 2710 default: 2711 /* if mq_mode is none, disable rss mode.*/ 2712 ngbe_rss_disable(dev); 2713 break; 2714 } 2715 } 2716 2717 return 0; 2718 } 2719 2720 void 2721 ngbe_set_rx_function(struct rte_eth_dev *dev) 2722 { 2723 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2724 2725 if (dev->data->scattered_rx) { 2726 /* 2727 * Set the scattered callback: there are bulk and 2728 * single allocation versions. 2729 */ 2730 if (adapter->rx_bulk_alloc_allowed) { 2731 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 2732 "allocation callback (port=%d).", 2733 dev->data->port_id); 2734 dev->rx_pkt_burst = ngbe_recv_pkts_sc_bulk_alloc; 2735 } else { 2736 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 2737 "single allocation) " 2738 "Scattered Rx callback " 2739 "(port=%d).", 2740 dev->data->port_id); 2741 2742 dev->rx_pkt_burst = ngbe_recv_pkts_sc_single_alloc; 2743 } 2744 /* 2745 * Below we set "simple" callbacks according to port/queues parameters. 2746 * If parameters allow we are going to choose between the following 2747 * callbacks: 2748 * - Bulk Allocation 2749 * - Single buffer allocation (the simplest one) 2750 */ 2751 } else if (adapter->rx_bulk_alloc_allowed) { 2752 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 2753 "satisfied. Rx Burst Bulk Alloc function " 2754 "will be used on port=%d.", 2755 dev->data->port_id); 2756 2757 dev->rx_pkt_burst = ngbe_recv_pkts_bulk_alloc; 2758 } else { 2759 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 2760 "satisfied, or Scattered Rx is requested " 2761 "(port=%d).", 2762 dev->data->port_id); 2763 2764 dev->rx_pkt_burst = ngbe_recv_pkts; 2765 } 2766 } 2767 2768 static const struct { 2769 eth_rx_burst_t pkt_burst; 2770 const char *info; 2771 } ngbe_rx_burst_infos[] = { 2772 { ngbe_recv_pkts_sc_single_alloc, "Scalar Scattered"}, 2773 { ngbe_recv_pkts_sc_bulk_alloc, "Scalar Scattered Bulk Alloc"}, 2774 { ngbe_recv_pkts_bulk_alloc, "Scalar Bulk Alloc"}, 2775 { ngbe_recv_pkts, "Scalar"}, 2776 }; 2777 2778 int 2779 ngbe_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 2780 struct rte_eth_burst_mode *mode) 2781 { 2782 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 2783 int ret = -EINVAL; 2784 unsigned int i; 2785 2786 for (i = 0; i < RTE_DIM(ngbe_rx_burst_infos); ++i) { 2787 if (pkt_burst == ngbe_rx_burst_infos[i].pkt_burst) { 2788 snprintf(mode->info, sizeof(mode->info), "%s", 2789 ngbe_rx_burst_infos[i].info); 2790 ret = 0; 2791 break; 2792 } 2793 } 2794 2795 return ret; 2796 } 2797 2798 /* 2799 * Initializes Receive Unit. 2800 */ 2801 int 2802 ngbe_dev_rx_init(struct rte_eth_dev *dev) 2803 { 2804 struct ngbe_hw *hw; 2805 struct ngbe_rx_queue *rxq; 2806 uint64_t bus_addr; 2807 uint32_t fctrl; 2808 uint32_t hlreg0; 2809 uint32_t srrctl; 2810 uint32_t rdrxctl; 2811 uint32_t rxcsum; 2812 uint16_t buf_size; 2813 uint16_t i; 2814 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 2815 2816 PMD_INIT_FUNC_TRACE(); 2817 hw = ngbe_dev_hw(dev); 2818 2819 /* 2820 * Make sure receives are disabled while setting 2821 * up the Rx context (registers, descriptor rings, etc.). 2822 */ 2823 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0); 2824 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0); 2825 2826 /* Enable receipt of broadcasted frames */ 2827 fctrl = rd32(hw, NGBE_PSRCTL); 2828 fctrl |= NGBE_PSRCTL_BCA; 2829 wr32(hw, NGBE_PSRCTL, fctrl); 2830 2831 /* 2832 * Configure CRC stripping, if any. 2833 */ 2834 hlreg0 = rd32(hw, NGBE_SECRXCTL); 2835 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2836 hlreg0 &= ~NGBE_SECRXCTL_CRCSTRIP; 2837 else 2838 hlreg0 |= NGBE_SECRXCTL_CRCSTRIP; 2839 hlreg0 &= ~NGBE_SECRXCTL_XDSA; 2840 wr32(hw, NGBE_SECRXCTL, hlreg0); 2841 2842 /* 2843 * Configure jumbo frame support, if any. 2844 */ 2845 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK, 2846 NGBE_FRMSZ_MAX(dev->data->mtu + NGBE_ETH_OVERHEAD)); 2847 2848 /* 2849 * If loopback mode is configured, set LPBK bit. 2850 */ 2851 hlreg0 = rd32(hw, NGBE_PSRCTL); 2852 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 2853 hlreg0 |= NGBE_PSRCTL_LBENA; 2854 else 2855 hlreg0 &= ~NGBE_PSRCTL_LBENA; 2856 2857 wr32(hw, NGBE_PSRCTL, hlreg0); 2858 2859 /* 2860 * Assume no header split and no VLAN strip support 2861 * on any Rx queue first . 2862 */ 2863 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2864 2865 /* Setup Rx queues */ 2866 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2867 rxq = dev->data->rx_queues[i]; 2868 2869 /* 2870 * Reset crc_len in case it was changed after queue setup by a 2871 * call to configure. 2872 */ 2873 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2874 rxq->crc_len = RTE_ETHER_CRC_LEN; 2875 else 2876 rxq->crc_len = 0; 2877 2878 /* Setup the Base and Length of the Rx Descriptor Rings */ 2879 bus_addr = rxq->rx_ring_phys_addr; 2880 wr32(hw, NGBE_RXBAL(rxq->reg_idx), 2881 (uint32_t)(bus_addr & BIT_MASK32)); 2882 wr32(hw, NGBE_RXBAH(rxq->reg_idx), 2883 (uint32_t)(bus_addr >> 32)); 2884 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 2885 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0); 2886 2887 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 2888 2889 /* Set if packets are dropped when no descriptors available */ 2890 if (rxq->drop_en) 2891 srrctl |= NGBE_RXCFG_DROP; 2892 2893 /* 2894 * Configure the Rx buffer size in the PKTLEN field of 2895 * the RXCFG register of the queue. 2896 * The value is in 1 KB resolution. Valid values can be from 2897 * 1 KB to 16 KB. 2898 */ 2899 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 2900 RTE_PKTMBUF_HEADROOM); 2901 buf_size = ROUND_DOWN(buf_size, 0x1 << 10); 2902 srrctl |= NGBE_RXCFG_PKTLEN(buf_size); 2903 2904 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl); 2905 2906 /* It adds dual VLAN length for supporting dual VLAN */ 2907 if (dev->data->mtu + NGBE_ETH_OVERHEAD + 2908 2 * RTE_VLAN_HLEN > buf_size) 2909 dev->data->scattered_rx = 1; 2910 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 2911 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2912 } 2913 2914 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 2915 dev->data->scattered_rx = 1; 2916 2917 /* 2918 * Device configured with multiple RX queues. 2919 */ 2920 ngbe_dev_mq_rx_configure(dev); 2921 2922 /* 2923 * Setup the Checksum Register. 2924 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 2925 * Enable IP/L4 checksum computation by hardware if requested to do so. 2926 */ 2927 rxcsum = rd32(hw, NGBE_PSRCTL); 2928 rxcsum |= NGBE_PSRCTL_PCSD; 2929 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 2930 rxcsum |= NGBE_PSRCTL_L4CSUM; 2931 else 2932 rxcsum &= ~NGBE_PSRCTL_L4CSUM; 2933 2934 wr32(hw, NGBE_PSRCTL, rxcsum); 2935 2936 if (hw->is_pf) { 2937 rdrxctl = rd32(hw, NGBE_SECRXCTL); 2938 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2939 rdrxctl &= ~NGBE_SECRXCTL_CRCSTRIP; 2940 else 2941 rdrxctl |= NGBE_SECRXCTL_CRCSTRIP; 2942 wr32(hw, NGBE_SECRXCTL, rdrxctl); 2943 } 2944 2945 ngbe_set_rx_function(dev); 2946 2947 return 0; 2948 } 2949 2950 /* 2951 * Initializes Transmit Unit. 2952 */ 2953 void 2954 ngbe_dev_tx_init(struct rte_eth_dev *dev) 2955 { 2956 struct ngbe_hw *hw; 2957 struct ngbe_tx_queue *txq; 2958 uint64_t bus_addr; 2959 uint16_t i; 2960 2961 PMD_INIT_FUNC_TRACE(); 2962 hw = ngbe_dev_hw(dev); 2963 2964 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA); 2965 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0); 2966 2967 /* Setup the Base and Length of the Tx Descriptor Rings */ 2968 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2969 txq = dev->data->tx_queues[i]; 2970 2971 bus_addr = txq->tx_ring_phys_addr; 2972 wr32(hw, NGBE_TXBAL(txq->reg_idx), 2973 (uint32_t)(bus_addr & BIT_MASK32)); 2974 wr32(hw, NGBE_TXBAH(txq->reg_idx), 2975 (uint32_t)(bus_addr >> 32)); 2976 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK, 2977 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 2978 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 2979 wr32(hw, NGBE_TXRP(txq->reg_idx), 0); 2980 wr32(hw, NGBE_TXWP(txq->reg_idx), 0); 2981 } 2982 } 2983 2984 /* 2985 * Set up link loopback mode Tx->Rx. 2986 */ 2987 static inline void 2988 ngbe_setup_loopback_link(struct ngbe_hw *hw) 2989 { 2990 PMD_INIT_FUNC_TRACE(); 2991 2992 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_LB, NGBE_MACRXCFG_LB); 2993 2994 msec_delay(50); 2995 } 2996 2997 /* 2998 * Start Transmit and Receive Units. 2999 */ 3000 int 3001 ngbe_dev_rxtx_start(struct rte_eth_dev *dev) 3002 { 3003 struct ngbe_hw *hw; 3004 struct ngbe_tx_queue *txq; 3005 struct ngbe_rx_queue *rxq; 3006 uint32_t dmatxctl; 3007 uint32_t rxctrl; 3008 uint16_t i; 3009 int ret = 0; 3010 3011 PMD_INIT_FUNC_TRACE(); 3012 hw = ngbe_dev_hw(dev); 3013 3014 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3015 txq = dev->data->tx_queues[i]; 3016 /* Setup Transmit Threshold Registers */ 3017 wr32m(hw, NGBE_TXCFG(txq->reg_idx), 3018 NGBE_TXCFG_HTHRESH_MASK | 3019 NGBE_TXCFG_WTHRESH_MASK, 3020 NGBE_TXCFG_HTHRESH(txq->hthresh) | 3021 NGBE_TXCFG_WTHRESH(txq->wthresh)); 3022 } 3023 3024 dmatxctl = rd32(hw, NGBE_DMATXCTRL); 3025 dmatxctl |= NGBE_DMATXCTRL_ENA; 3026 wr32(hw, NGBE_DMATXCTRL, dmatxctl); 3027 3028 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3029 txq = dev->data->tx_queues[i]; 3030 if (txq->tx_deferred_start == 0) { 3031 ret = ngbe_dev_tx_queue_start(dev, i); 3032 if (ret < 0) 3033 return ret; 3034 } 3035 } 3036 3037 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3038 rxq = dev->data->rx_queues[i]; 3039 if (rxq->rx_deferred_start == 0) { 3040 ret = ngbe_dev_rx_queue_start(dev, i); 3041 if (ret < 0) 3042 return ret; 3043 } 3044 } 3045 3046 /* Enable Receive engine */ 3047 rxctrl = rd32(hw, NGBE_PBRXCTL); 3048 rxctrl |= NGBE_PBRXCTL_ENA; 3049 hw->mac.enable_rx_dma(hw, rxctrl); 3050 3051 /* If loopback mode is enabled, set up the link accordingly */ 3052 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 3053 ngbe_setup_loopback_link(hw); 3054 3055 return 0; 3056 } 3057 3058 void 3059 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 3060 { 3061 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 3062 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id)); 3063 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id)); 3064 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id)); 3065 } 3066 3067 void 3068 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 3069 { 3070 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 3071 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++)); 3072 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++)); 3073 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA); 3074 } 3075 3076 void 3077 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 3078 { 3079 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 3080 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id)); 3081 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id)); 3082 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id)); 3083 } 3084 3085 void 3086 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 3087 { 3088 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 3089 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++)); 3090 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++)); 3091 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA); 3092 } 3093 3094 /* 3095 * Start Receive Units for specified queue. 3096 */ 3097 int 3098 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3099 { 3100 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3101 struct ngbe_rx_queue *rxq; 3102 uint32_t rxdctl; 3103 int poll_ms; 3104 3105 PMD_INIT_FUNC_TRACE(); 3106 3107 rxq = dev->data->rx_queues[rx_queue_id]; 3108 3109 /* Allocate buffers for descriptor rings */ 3110 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) { 3111 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 3112 rx_queue_id); 3113 return -1; 3114 } 3115 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3116 rxdctl |= NGBE_RXCFG_ENA; 3117 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl); 3118 3119 /* Wait until Rx Enable ready */ 3120 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3121 do { 3122 rte_delay_ms(1); 3123 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3124 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA)); 3125 if (poll_ms == 0) 3126 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 3127 rte_wmb(); 3128 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 3129 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 3130 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3131 3132 return 0; 3133 } 3134 3135 /* 3136 * Stop Receive Units for specified queue. 3137 */ 3138 int 3139 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3140 { 3141 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3142 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 3143 struct ngbe_rx_queue *rxq; 3144 uint32_t rxdctl; 3145 int poll_ms; 3146 3147 PMD_INIT_FUNC_TRACE(); 3148 3149 rxq = dev->data->rx_queues[rx_queue_id]; 3150 3151 ngbe_dev_save_rx_queue(hw, rxq->reg_idx); 3152 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0); 3153 3154 /* Wait until Rx Enable bit clear */ 3155 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3156 do { 3157 rte_delay_ms(1); 3158 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3159 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA)); 3160 if (poll_ms == 0) 3161 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 3162 3163 rte_delay_us(RTE_NGBE_WAIT_100_US); 3164 ngbe_dev_store_rx_queue(hw, rxq->reg_idx); 3165 3166 ngbe_rx_queue_release_mbufs(rxq); 3167 ngbe_reset_rx_queue(adapter, rxq); 3168 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3169 3170 return 0; 3171 } 3172 3173 /* 3174 * Start Transmit Units for specified queue. 3175 */ 3176 int 3177 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3178 { 3179 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3180 struct ngbe_tx_queue *txq; 3181 uint32_t txdctl; 3182 int poll_ms; 3183 3184 PMD_INIT_FUNC_TRACE(); 3185 3186 txq = dev->data->tx_queues[tx_queue_id]; 3187 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA); 3188 3189 /* Wait until Tx Enable ready */ 3190 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3191 do { 3192 rte_delay_ms(1); 3193 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3194 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA)); 3195 if (poll_ms == 0) 3196 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 3197 tx_queue_id); 3198 3199 rte_wmb(); 3200 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail); 3201 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3202 3203 return 0; 3204 } 3205 3206 /* 3207 * Stop Transmit Units for specified queue. 3208 */ 3209 int 3210 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3211 { 3212 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3213 struct ngbe_tx_queue *txq; 3214 uint32_t txdctl; 3215 uint32_t txtdh, txtdt; 3216 int poll_ms; 3217 3218 PMD_INIT_FUNC_TRACE(); 3219 3220 txq = dev->data->tx_queues[tx_queue_id]; 3221 3222 /* Wait until Tx queue is empty */ 3223 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3224 do { 3225 rte_delay_us(RTE_NGBE_WAIT_100_US); 3226 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx)); 3227 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx)); 3228 } while (--poll_ms && (txtdh != txtdt)); 3229 if (poll_ms == 0) 3230 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.", 3231 tx_queue_id); 3232 3233 ngbe_dev_save_tx_queue(hw, txq->reg_idx); 3234 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0); 3235 3236 /* Wait until Tx Enable bit clear */ 3237 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3238 do { 3239 rte_delay_ms(1); 3240 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3241 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA)); 3242 if (poll_ms == 0) 3243 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 3244 tx_queue_id); 3245 3246 rte_delay_us(RTE_NGBE_WAIT_100_US); 3247 ngbe_dev_store_tx_queue(hw, txq->reg_idx); 3248 3249 if (txq->ops != NULL) { 3250 txq->ops->release_mbufs(txq); 3251 txq->ops->reset(txq); 3252 } 3253 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3254 3255 return 0; 3256 } 3257 3258 void 3259 ngbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3260 struct rte_eth_rxq_info *qinfo) 3261 { 3262 struct ngbe_rx_queue *rxq; 3263 3264 rxq = dev->data->rx_queues[queue_id]; 3265 3266 qinfo->mp = rxq->mb_pool; 3267 qinfo->scattered_rx = dev->data->scattered_rx; 3268 qinfo->nb_desc = rxq->nb_rx_desc; 3269 3270 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 3271 qinfo->conf.rx_drop_en = rxq->drop_en; 3272 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 3273 qinfo->conf.offloads = rxq->offloads; 3274 } 3275 3276 void 3277 ngbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3278 struct rte_eth_txq_info *qinfo) 3279 { 3280 struct ngbe_tx_queue *txq; 3281 3282 txq = dev->data->tx_queues[queue_id]; 3283 3284 qinfo->nb_desc = txq->nb_tx_desc; 3285 3286 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 3287 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 3288 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 3289 3290 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 3291 qinfo->conf.offloads = txq->offloads; 3292 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 3293 } 3294