1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdint.h> 9 #include <rte_ethdev.h> 10 #include <ethdev_driver.h> 11 #include <rte_malloc.h> 12 #include <rte_net.h> 13 14 #include "ngbe_logs.h" 15 #include "base/ngbe.h" 16 #include "ngbe_ethdev.h" 17 #include "ngbe_rxtx.h" 18 19 #ifdef RTE_LIBRTE_IEEE1588 20 #define NGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST 21 #else 22 #define NGBE_TX_IEEE1588_TMST 0 23 #endif 24 25 /* Bit Mask to indicate what bits required for building Tx context */ 26 static const u64 NGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM | 27 RTE_MBUF_F_TX_OUTER_IPV6 | 28 RTE_MBUF_F_TX_OUTER_IPV4 | 29 RTE_MBUF_F_TX_IPV6 | 30 RTE_MBUF_F_TX_IPV4 | 31 RTE_MBUF_F_TX_VLAN | 32 RTE_MBUF_F_TX_L4_MASK | 33 RTE_MBUF_F_TX_TCP_SEG | 34 RTE_MBUF_F_TX_TUNNEL_MASK | 35 RTE_MBUF_F_TX_OUTER_IP_CKSUM | 36 NGBE_TX_IEEE1588_TMST); 37 38 #define NGBE_TX_OFFLOAD_NOTSUP_MASK \ 39 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ NGBE_TX_OFFLOAD_MASK) 40 41 /* 42 * Prefetch a cache line into all cache levels. 43 */ 44 #define rte_ngbe_prefetch(p) rte_prefetch0(p) 45 46 /********************************************************************* 47 * 48 * Tx functions 49 * 50 **********************************************************************/ 51 52 /* 53 * Check for descriptors with their DD bit set and free mbufs. 54 * Return the total number of buffers freed. 55 */ 56 static __rte_always_inline int 57 ngbe_tx_free_bufs(struct ngbe_tx_queue *txq) 58 { 59 struct ngbe_tx_entry *txep; 60 uint32_t status; 61 int i, nb_free = 0; 62 struct rte_mbuf *m, *free[RTE_NGBE_TX_MAX_FREE_BUF_SZ]; 63 64 /* check DD bit on threshold descriptor */ 65 status = txq->tx_ring[txq->tx_next_dd].dw3; 66 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 67 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 68 ngbe_set32_masked(txq->tdc_reg_addr, 69 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 70 return 0; 71 } 72 73 /* 74 * first buffer to free from S/W ring is at index 75 * tx_next_dd - (tx_free_thresh-1) 76 */ 77 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 78 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 79 /* free buffers one at a time */ 80 m = rte_pktmbuf_prefree_seg(txep->mbuf); 81 txep->mbuf = NULL; 82 83 if (unlikely(m == NULL)) 84 continue; 85 86 if (nb_free >= RTE_NGBE_TX_MAX_FREE_BUF_SZ || 87 (nb_free > 0 && m->pool != free[0]->pool)) { 88 rte_mempool_put_bulk(free[0]->pool, 89 (void **)free, nb_free); 90 nb_free = 0; 91 } 92 93 free[nb_free++] = m; 94 } 95 96 if (nb_free > 0) 97 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 98 99 /* buffers were freed, update counters */ 100 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 101 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 102 if (txq->tx_next_dd >= txq->nb_tx_desc) 103 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 104 105 return txq->tx_free_thresh; 106 } 107 108 /* Populate 4 descriptors with data from 4 mbufs */ 109 static inline void 110 tx4(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 111 { 112 uint64_t buf_dma_addr; 113 uint32_t pkt_len; 114 int i; 115 116 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 117 buf_dma_addr = rte_mbuf_data_iova(*pkts); 118 pkt_len = (*pkts)->data_len; 119 120 /* write data to descriptor */ 121 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 122 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 123 NGBE_TXD_DATLEN(pkt_len)); 124 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 125 126 rte_prefetch0(&(*pkts)->pool); 127 } 128 } 129 130 /* Populate 1 descriptor with data from 1 mbuf */ 131 static inline void 132 tx1(volatile struct ngbe_tx_desc *txdp, struct rte_mbuf **pkts) 133 { 134 uint64_t buf_dma_addr; 135 uint32_t pkt_len; 136 137 buf_dma_addr = rte_mbuf_data_iova(*pkts); 138 pkt_len = (*pkts)->data_len; 139 140 /* write data to descriptor */ 141 txdp->qw0 = cpu_to_le64(buf_dma_addr); 142 txdp->dw2 = cpu_to_le32(NGBE_TXD_FLAGS | 143 NGBE_TXD_DATLEN(pkt_len)); 144 txdp->dw3 = cpu_to_le32(NGBE_TXD_PAYLEN(pkt_len)); 145 146 rte_prefetch0(&(*pkts)->pool); 147 } 148 149 /* 150 * Fill H/W descriptor ring with mbuf data. 151 * Copy mbuf pointers to the S/W ring. 152 */ 153 static inline void 154 ngbe_tx_fill_hw_ring(struct ngbe_tx_queue *txq, struct rte_mbuf **pkts, 155 uint16_t nb_pkts) 156 { 157 volatile struct ngbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 158 struct ngbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 159 const int N_PER_LOOP = 4; 160 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 161 int mainpart, leftover; 162 int i, j; 163 164 /* 165 * Process most of the packets in chunks of N pkts. Any 166 * leftover packets will get processed one at a time. 167 */ 168 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 169 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 170 for (i = 0; i < mainpart; i += N_PER_LOOP) { 171 /* Copy N mbuf pointers to the S/W ring */ 172 for (j = 0; j < N_PER_LOOP; ++j) 173 (txep + i + j)->mbuf = *(pkts + i + j); 174 tx4(txdp + i, pkts + i); 175 } 176 177 if (unlikely(leftover > 0)) { 178 for (i = 0; i < leftover; ++i) { 179 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 180 tx1(txdp + mainpart + i, pkts + mainpart + i); 181 } 182 } 183 } 184 185 static inline uint16_t 186 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 187 uint16_t nb_pkts) 188 { 189 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 190 uint16_t n = 0; 191 192 /* 193 * Begin scanning the H/W ring for done descriptors when the 194 * number of available descriptors drops below tx_free_thresh. 195 * For each done descriptor, free the associated buffer. 196 */ 197 if (txq->nb_tx_free < txq->tx_free_thresh) 198 ngbe_tx_free_bufs(txq); 199 200 /* Only use descriptors that are available */ 201 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 202 if (unlikely(nb_pkts == 0)) 203 return 0; 204 205 /* Use exactly nb_pkts descriptors */ 206 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 207 208 /* 209 * At this point, we know there are enough descriptors in the 210 * ring to transmit all the packets. This assumes that each 211 * mbuf contains a single segment, and that no new offloads 212 * are expected, which would require a new context descriptor. 213 */ 214 215 /* 216 * See if we're going to wrap-around. If so, handle the top 217 * of the descriptor ring first, then do the bottom. If not, 218 * the processing looks just like the "bottom" part anyway... 219 */ 220 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 221 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 222 ngbe_tx_fill_hw_ring(txq, tx_pkts, n); 223 txq->tx_tail = 0; 224 } 225 226 /* Fill H/W descriptor ring with mbuf data */ 227 ngbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 228 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 229 230 /* 231 * Check for wrap-around. This would only happen if we used 232 * up to the last descriptor in the ring, no more, no less. 233 */ 234 if (txq->tx_tail >= txq->nb_tx_desc) 235 txq->tx_tail = 0; 236 237 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 238 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 239 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 240 241 /* update tail pointer */ 242 rte_wmb(); 243 ngbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 244 245 return nb_pkts; 246 } 247 248 uint16_t 249 ngbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 250 uint16_t nb_pkts) 251 { 252 uint16_t nb_tx; 253 254 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 255 if (likely(nb_pkts <= RTE_PMD_NGBE_TX_MAX_BURST)) 256 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 257 258 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 259 nb_tx = 0; 260 while (nb_pkts != 0) { 261 uint16_t ret, n; 262 263 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_TX_MAX_BURST); 264 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 265 nb_tx = (uint16_t)(nb_tx + ret); 266 nb_pkts = (uint16_t)(nb_pkts - ret); 267 if (ret < n) 268 break; 269 } 270 271 return nb_tx; 272 } 273 274 static inline void 275 ngbe_set_xmit_ctx(struct ngbe_tx_queue *txq, 276 volatile struct ngbe_tx_ctx_desc *ctx_txd, 277 uint64_t ol_flags, union ngbe_tx_offload tx_offload) 278 { 279 union ngbe_tx_offload tx_offload_mask; 280 uint32_t type_tucmd_mlhl; 281 uint32_t mss_l4len_idx; 282 uint32_t ctx_idx; 283 uint32_t vlan_macip_lens; 284 uint32_t tunnel_seed; 285 286 ctx_idx = txq->ctx_curr; 287 tx_offload_mask.data[0] = 0; 288 tx_offload_mask.data[1] = 0; 289 290 /* Specify which HW CTX to upload. */ 291 mss_l4len_idx = NGBE_TXD_IDX(ctx_idx); 292 type_tucmd_mlhl = NGBE_TXD_CTXT; 293 294 tx_offload_mask.ptid |= ~0; 295 type_tucmd_mlhl |= NGBE_TXD_PTID(tx_offload.ptid); 296 297 /* check if TCP segmentation required for this packet */ 298 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 299 tx_offload_mask.l2_len |= ~0; 300 tx_offload_mask.l3_len |= ~0; 301 tx_offload_mask.l4_len |= ~0; 302 tx_offload_mask.tso_segsz |= ~0; 303 mss_l4len_idx |= NGBE_TXD_MSS(tx_offload.tso_segsz); 304 mss_l4len_idx |= NGBE_TXD_L4LEN(tx_offload.l4_len); 305 } else { /* no TSO, check if hardware checksum is needed */ 306 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 307 tx_offload_mask.l2_len |= ~0; 308 tx_offload_mask.l3_len |= ~0; 309 } 310 311 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 312 case RTE_MBUF_F_TX_UDP_CKSUM: 313 mss_l4len_idx |= 314 NGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr)); 315 tx_offload_mask.l2_len |= ~0; 316 tx_offload_mask.l3_len |= ~0; 317 break; 318 case RTE_MBUF_F_TX_TCP_CKSUM: 319 mss_l4len_idx |= 320 NGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr)); 321 tx_offload_mask.l2_len |= ~0; 322 tx_offload_mask.l3_len |= ~0; 323 break; 324 case RTE_MBUF_F_TX_SCTP_CKSUM: 325 mss_l4len_idx |= 326 NGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr)); 327 tx_offload_mask.l2_len |= ~0; 328 tx_offload_mask.l3_len |= ~0; 329 break; 330 default: 331 break; 332 } 333 } 334 335 vlan_macip_lens = NGBE_TXD_IPLEN(tx_offload.l3_len >> 1); 336 337 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 338 tx_offload_mask.outer_tun_len |= ~0; 339 tx_offload_mask.outer_l2_len |= ~0; 340 tx_offload_mask.outer_l3_len |= ~0; 341 tx_offload_mask.l2_len |= ~0; 342 tunnel_seed = NGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1); 343 tunnel_seed |= NGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2); 344 345 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 346 case RTE_MBUF_F_TX_TUNNEL_IPIP: 347 /* for non UDP / GRE tunneling, set to 0b */ 348 break; 349 default: 350 PMD_TX_LOG(ERR, "Tunnel type not supported"); 351 return; 352 } 353 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.outer_l2_len); 354 } else { 355 tunnel_seed = 0; 356 vlan_macip_lens |= NGBE_TXD_MACLEN(tx_offload.l2_len); 357 } 358 359 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 360 tx_offload_mask.vlan_tci |= ~0; 361 vlan_macip_lens |= NGBE_TXD_VLAN(tx_offload.vlan_tci); 362 } 363 364 txq->ctx_cache[ctx_idx].flags = ol_flags; 365 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 366 tx_offload_mask.data[0] & tx_offload.data[0]; 367 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 368 tx_offload_mask.data[1] & tx_offload.data[1]; 369 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 370 371 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens); 372 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed); 373 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl); 374 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx); 375 } 376 377 /* 378 * Check which hardware context can be used. Use the existing match 379 * or create a new context descriptor. 380 */ 381 static inline uint32_t 382 what_ctx_update(struct ngbe_tx_queue *txq, uint64_t flags, 383 union ngbe_tx_offload tx_offload) 384 { 385 /* If match with the current used context */ 386 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 387 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 388 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 389 & tx_offload.data[0])) && 390 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 391 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 392 & tx_offload.data[1])))) 393 return txq->ctx_curr; 394 395 /* What if match with the next context */ 396 txq->ctx_curr ^= 1; 397 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 398 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 399 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 400 & tx_offload.data[0])) && 401 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 402 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 403 & tx_offload.data[1])))) 404 return txq->ctx_curr; 405 406 /* Mismatch, use the previous context */ 407 return NGBE_CTX_NUM; 408 } 409 410 static inline uint32_t 411 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 412 { 413 uint32_t tmp = 0; 414 415 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) { 416 tmp |= NGBE_TXD_CC; 417 tmp |= NGBE_TXD_L4CS; 418 } 419 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 420 tmp |= NGBE_TXD_CC; 421 tmp |= NGBE_TXD_IPCS; 422 } 423 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 424 tmp |= NGBE_TXD_CC; 425 tmp |= NGBE_TXD_EIPCS; 426 } 427 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 428 tmp |= NGBE_TXD_CC; 429 /* implies IPv4 cksum */ 430 if (ol_flags & RTE_MBUF_F_TX_IPV4) 431 tmp |= NGBE_TXD_IPCS; 432 tmp |= NGBE_TXD_L4CS; 433 } 434 if (ol_flags & RTE_MBUF_F_TX_VLAN) 435 tmp |= NGBE_TXD_CC; 436 437 return tmp; 438 } 439 440 static inline uint32_t 441 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 442 { 443 uint32_t cmdtype = 0; 444 445 if (ol_flags & RTE_MBUF_F_TX_VLAN) 446 cmdtype |= NGBE_TXD_VLE; 447 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 448 cmdtype |= NGBE_TXD_TSE; 449 return cmdtype; 450 } 451 452 static inline uint8_t 453 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) 454 { 455 bool tun; 456 457 if (ptype) 458 return ngbe_encode_ptype(ptype); 459 460 /* Only support flags in NGBE_TX_OFFLOAD_MASK */ 461 tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK); 462 463 /* L2 level */ 464 ptype = RTE_PTYPE_L2_ETHER; 465 if (oflags & RTE_MBUF_F_TX_VLAN) 466 ptype |= RTE_PTYPE_L2_ETHER_VLAN; 467 468 /* L3 level */ 469 if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM)) 470 ptype |= RTE_PTYPE_L3_IPV4; 471 else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6)) 472 ptype |= RTE_PTYPE_L3_IPV6; 473 474 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM)) 475 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4); 476 else if (oflags & (RTE_MBUF_F_TX_IPV6)) 477 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6); 478 479 /* L4 level */ 480 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) { 481 case RTE_MBUF_F_TX_TCP_CKSUM: 482 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 483 break; 484 case RTE_MBUF_F_TX_UDP_CKSUM: 485 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP); 486 break; 487 case RTE_MBUF_F_TX_SCTP_CKSUM: 488 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP); 489 break; 490 } 491 492 if (oflags & RTE_MBUF_F_TX_TCP_SEG) 493 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 494 495 /* Tunnel */ 496 switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) { 497 case RTE_MBUF_F_TX_TUNNEL_IPIP: 498 case RTE_MBUF_F_TX_TUNNEL_IP: 499 ptype |= RTE_PTYPE_L2_ETHER | 500 RTE_PTYPE_L3_IPV4 | 501 RTE_PTYPE_TUNNEL_IP; 502 break; 503 } 504 505 return ngbe_encode_ptype(ptype); 506 } 507 508 /* Reset transmit descriptors after they have been used */ 509 static inline int 510 ngbe_xmit_cleanup(struct ngbe_tx_queue *txq) 511 { 512 struct ngbe_tx_entry *sw_ring = txq->sw_ring; 513 volatile struct ngbe_tx_desc *txr = txq->tx_ring; 514 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 515 uint16_t nb_tx_desc = txq->nb_tx_desc; 516 uint16_t desc_to_clean_to; 517 uint16_t nb_tx_to_clean; 518 uint32_t status; 519 520 /* Determine the last descriptor needing to be cleaned */ 521 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh); 522 if (desc_to_clean_to >= nb_tx_desc) 523 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 524 525 /* Check to make sure the last descriptor to clean is done */ 526 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 527 status = txr[desc_to_clean_to].dw3; 528 if (!(status & rte_cpu_to_le_32(NGBE_TXD_DD))) { 529 PMD_TX_LOG(DEBUG, 530 "Tx descriptor %4u is not done" 531 "(port=%d queue=%d)", 532 desc_to_clean_to, 533 txq->port_id, txq->queue_id); 534 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 535 ngbe_set32_masked(txq->tdc_reg_addr, 536 NGBE_TXCFG_FLUSH, NGBE_TXCFG_FLUSH); 537 /* Failed to clean any descriptors, better luck next time */ 538 return -(1); 539 } 540 541 /* Figure out how many descriptors will be cleaned */ 542 if (last_desc_cleaned > desc_to_clean_to) 543 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 544 desc_to_clean_to); 545 else 546 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 547 last_desc_cleaned); 548 549 PMD_TX_LOG(DEBUG, 550 "Cleaning %4u Tx descriptors: %4u to %4u (port=%d queue=%d)", 551 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 552 txq->port_id, txq->queue_id); 553 554 /* 555 * The last descriptor to clean is done, so that means all the 556 * descriptors from the last descriptor that was cleaned 557 * up to the last descriptor with the RS bit set 558 * are done. Only reset the threshold descriptor. 559 */ 560 txr[desc_to_clean_to].dw3 = 0; 561 562 /* Update the txq to reflect the last descriptor that was cleaned */ 563 txq->last_desc_cleaned = desc_to_clean_to; 564 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 565 566 /* No Error */ 567 return 0; 568 } 569 570 uint16_t 571 ngbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 572 uint16_t nb_pkts) 573 { 574 struct ngbe_tx_queue *txq; 575 struct ngbe_tx_entry *sw_ring; 576 struct ngbe_tx_entry *txe, *txn; 577 volatile struct ngbe_tx_desc *txr; 578 volatile struct ngbe_tx_desc *txd; 579 struct rte_mbuf *tx_pkt; 580 struct rte_mbuf *m_seg; 581 uint64_t buf_dma_addr; 582 uint32_t olinfo_status; 583 uint32_t cmd_type_len; 584 uint32_t pkt_len; 585 uint16_t slen; 586 uint64_t ol_flags; 587 uint16_t tx_id; 588 uint16_t tx_last; 589 uint16_t nb_tx; 590 uint16_t nb_used; 591 uint64_t tx_ol_req; 592 uint32_t ctx = 0; 593 uint32_t new_ctx; 594 union ngbe_tx_offload tx_offload; 595 596 tx_offload.data[0] = 0; 597 tx_offload.data[1] = 0; 598 txq = tx_queue; 599 sw_ring = txq->sw_ring; 600 txr = txq->tx_ring; 601 tx_id = txq->tx_tail; 602 txe = &sw_ring[tx_id]; 603 604 /* Determine if the descriptor ring needs to be cleaned. */ 605 if (txq->nb_tx_free < txq->tx_free_thresh) 606 ngbe_xmit_cleanup(txq); 607 608 rte_prefetch0(&txe->mbuf->pool); 609 610 /* Tx loop */ 611 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 612 new_ctx = 0; 613 tx_pkt = *tx_pkts++; 614 pkt_len = tx_pkt->pkt_len; 615 616 /* 617 * Determine how many (if any) context descriptors 618 * are needed for offload functionality. 619 */ 620 ol_flags = tx_pkt->ol_flags; 621 622 /* If hardware offload required */ 623 tx_ol_req = ol_flags & NGBE_TX_OFFLOAD_MASK; 624 if (tx_ol_req) { 625 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req, 626 tx_pkt->packet_type); 627 tx_offload.l2_len = tx_pkt->l2_len; 628 tx_offload.l3_len = tx_pkt->l3_len; 629 tx_offload.l4_len = tx_pkt->l4_len; 630 tx_offload.vlan_tci = tx_pkt->vlan_tci; 631 tx_offload.tso_segsz = tx_pkt->tso_segsz; 632 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 633 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 634 tx_offload.outer_tun_len = 0; 635 636 /* If new context need be built or reuse the exist ctx*/ 637 ctx = what_ctx_update(txq, tx_ol_req, tx_offload); 638 /* Only allocate context descriptor if required */ 639 new_ctx = (ctx == NGBE_CTX_NUM); 640 ctx = txq->ctx_curr; 641 } 642 643 /* 644 * Keep track of how many descriptors are used this loop 645 * This will always be the number of segments + the number of 646 * Context descriptors required to transmit the packet 647 */ 648 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 649 650 /* 651 * The number of descriptors that must be allocated for a 652 * packet is the number of segments of that packet, plus 1 653 * Context Descriptor for the hardware offload, if any. 654 * Determine the last Tx descriptor to allocate in the Tx ring 655 * for the packet, starting from the current position (tx_id) 656 * in the ring. 657 */ 658 tx_last = (uint16_t)(tx_id + nb_used - 1); 659 660 /* Circular ring */ 661 if (tx_last >= txq->nb_tx_desc) 662 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 663 664 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 665 " tx_first=%u tx_last=%u", 666 (uint16_t)txq->port_id, 667 (uint16_t)txq->queue_id, 668 (uint32_t)pkt_len, 669 (uint16_t)tx_id, 670 (uint16_t)tx_last); 671 672 /* 673 * Make sure there are enough Tx descriptors available to 674 * transmit the entire packet. 675 * nb_used better be less than or equal to txq->tx_free_thresh 676 */ 677 if (nb_used > txq->nb_tx_free) { 678 PMD_TX_LOG(DEBUG, 679 "Not enough free Tx descriptors " 680 "nb_used=%4u nb_free=%4u " 681 "(port=%d queue=%d)", 682 nb_used, txq->nb_tx_free, 683 txq->port_id, txq->queue_id); 684 685 if (ngbe_xmit_cleanup(txq) != 0) { 686 /* Could not clean any descriptors */ 687 if (nb_tx == 0) 688 return 0; 689 goto end_of_tx; 690 } 691 692 /* nb_used better be <= txq->tx_free_thresh */ 693 if (unlikely(nb_used > txq->tx_free_thresh)) { 694 PMD_TX_LOG(DEBUG, 695 "The number of descriptors needed to " 696 "transmit the packet exceeds the " 697 "RS bit threshold. This will impact " 698 "performance." 699 "nb_used=%4u nb_free=%4u " 700 "tx_free_thresh=%4u. " 701 "(port=%d queue=%d)", 702 nb_used, txq->nb_tx_free, 703 txq->tx_free_thresh, 704 txq->port_id, txq->queue_id); 705 /* 706 * Loop here until there are enough Tx 707 * descriptors or until the ring cannot be 708 * cleaned. 709 */ 710 while (nb_used > txq->nb_tx_free) { 711 if (ngbe_xmit_cleanup(txq) != 0) { 712 /* 713 * Could not clean any 714 * descriptors 715 */ 716 if (nb_tx == 0) 717 return 0; 718 goto end_of_tx; 719 } 720 } 721 } 722 } 723 724 /* 725 * By now there are enough free Tx descriptors to transmit 726 * the packet. 727 */ 728 729 /* 730 * Set common flags of all Tx Data Descriptors. 731 * 732 * The following bits must be set in the first Data Descriptor 733 * and are ignored in the other ones: 734 * - NGBE_TXD_FCS 735 * 736 * The following bits must only be set in the last Data 737 * Descriptor: 738 * - NGBE_TXD_EOP 739 */ 740 cmd_type_len = NGBE_TXD_FCS; 741 742 #ifdef RTE_LIBRTE_IEEE1588 743 if (ol_flags & PKT_TX_IEEE1588_TMST) 744 cmd_type_len |= NGBE_TXD_1588; 745 #endif 746 747 olinfo_status = 0; 748 if (tx_ol_req) { 749 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 750 /* when TSO is on, paylen in descriptor is the 751 * not the packet len but the tcp payload len 752 */ 753 pkt_len -= (tx_offload.l2_len + 754 tx_offload.l3_len + tx_offload.l4_len); 755 pkt_len -= 756 (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) 757 ? tx_offload.outer_l2_len + 758 tx_offload.outer_l3_len : 0; 759 } 760 761 /* 762 * Setup the Tx Context Descriptor if required 763 */ 764 if (new_ctx) { 765 volatile struct ngbe_tx_ctx_desc *ctx_txd; 766 767 ctx_txd = (volatile struct ngbe_tx_ctx_desc *) 768 &txr[tx_id]; 769 770 txn = &sw_ring[txe->next_id]; 771 rte_prefetch0(&txn->mbuf->pool); 772 773 if (txe->mbuf != NULL) { 774 rte_pktmbuf_free_seg(txe->mbuf); 775 txe->mbuf = NULL; 776 } 777 778 ngbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 779 tx_offload); 780 781 txe->last_id = tx_last; 782 tx_id = txe->next_id; 783 txe = txn; 784 } 785 786 /* 787 * Setup the Tx Data Descriptor, 788 * This path will go through 789 * whatever new/reuse the context descriptor 790 */ 791 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 792 olinfo_status |= 793 tx_desc_cksum_flags_to_olinfo(ol_flags); 794 olinfo_status |= NGBE_TXD_IDX(ctx); 795 } 796 797 olinfo_status |= NGBE_TXD_PAYLEN(pkt_len); 798 799 m_seg = tx_pkt; 800 do { 801 txd = &txr[tx_id]; 802 txn = &sw_ring[txe->next_id]; 803 rte_prefetch0(&txn->mbuf->pool); 804 805 if (txe->mbuf != NULL) 806 rte_pktmbuf_free_seg(txe->mbuf); 807 txe->mbuf = m_seg; 808 809 /* 810 * Set up Transmit Data Descriptor. 811 */ 812 slen = m_seg->data_len; 813 buf_dma_addr = rte_mbuf_data_iova(m_seg); 814 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr); 815 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen); 816 txd->dw3 = rte_cpu_to_le_32(olinfo_status); 817 txe->last_id = tx_last; 818 tx_id = txe->next_id; 819 txe = txn; 820 m_seg = m_seg->next; 821 } while (m_seg != NULL); 822 823 /* 824 * The last packet data descriptor needs End Of Packet (EOP) 825 */ 826 cmd_type_len |= NGBE_TXD_EOP; 827 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 828 829 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len); 830 } 831 832 end_of_tx: 833 834 rte_wmb(); 835 836 /* 837 * Set the Transmit Descriptor Tail (TDT) 838 */ 839 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 840 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 841 (uint16_t)tx_id, (uint16_t)nb_tx); 842 ngbe_set32_relaxed(txq->tdt_reg_addr, tx_id); 843 txq->tx_tail = tx_id; 844 845 return nb_tx; 846 } 847 848 /********************************************************************* 849 * 850 * Tx prep functions 851 * 852 **********************************************************************/ 853 uint16_t 854 ngbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 855 { 856 int i, ret; 857 uint64_t ol_flags; 858 struct rte_mbuf *m; 859 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 860 861 for (i = 0; i < nb_pkts; i++) { 862 m = tx_pkts[i]; 863 ol_flags = m->ol_flags; 864 865 /** 866 * Check if packet meets requirements for number of segments 867 * 868 * NOTE: for ngbe it's always (40 - WTHRESH) for both TSO and 869 * non-TSO 870 */ 871 872 if (m->nb_segs > NGBE_TX_MAX_SEG - txq->wthresh) { 873 rte_errno = -EINVAL; 874 return i; 875 } 876 877 if (ol_flags & NGBE_TX_OFFLOAD_NOTSUP_MASK) { 878 rte_errno = -ENOTSUP; 879 return i; 880 } 881 882 #ifdef RTE_ETHDEV_DEBUG_TX 883 ret = rte_validate_tx_offload(m); 884 if (ret != 0) { 885 rte_errno = ret; 886 return i; 887 } 888 #endif 889 ret = rte_net_intel_cksum_prepare(m); 890 if (ret != 0) { 891 rte_errno = ret; 892 return i; 893 } 894 } 895 896 return i; 897 } 898 899 /********************************************************************* 900 * 901 * Rx functions 902 * 903 **********************************************************************/ 904 static inline uint32_t 905 ngbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) 906 { 907 uint16_t ptid = NGBE_RXD_PTID(pkt_info); 908 909 ptid &= ptid_mask; 910 911 return ngbe_decode_ptype(ptid); 912 } 913 914 static inline uint64_t 915 ngbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) 916 { 917 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { 918 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 919 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 920 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 921 0, 0, 0, RTE_MBUF_F_RX_FDIR, 922 }; 923 #ifdef RTE_LIBRTE_IEEE1588 924 static uint64_t ip_pkt_etqf_map[8] = { 925 0, 0, 0, PKT_RX_IEEE1588_PTP, 926 0, 0, 0, 0, 927 }; 928 int etfid = ngbe_etflt_id(NGBE_RXD_PTID(pkt_info)); 929 if (likely(-1 != etfid)) 930 return ip_pkt_etqf_map[etfid] | 931 ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 932 else 933 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 934 #else 935 return ip_rss_types_map[NGBE_RXD_RSSTYPE(pkt_info)]; 936 #endif 937 } 938 939 static inline uint64_t 940 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 941 { 942 uint64_t pkt_flags; 943 944 /* 945 * Check if VLAN present only. 946 * Do not check whether L3/L4 rx checksum done by NIC or not, 947 * That can be found from rte_eth_rxmode.offloads flag 948 */ 949 pkt_flags = (rx_status & NGBE_RXD_STAT_VLAN && 950 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED) 951 ? vlan_flags : 0; 952 953 #ifdef RTE_LIBRTE_IEEE1588 954 if (rx_status & NGBE_RXD_STAT_1588) 955 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST; 956 #endif 957 return pkt_flags; 958 } 959 960 static inline uint64_t 961 rx_desc_error_to_pkt_flags(uint32_t rx_status) 962 { 963 uint64_t pkt_flags = 0; 964 965 /* checksum offload can't be disabled */ 966 if (rx_status & NGBE_RXD_STAT_IPCS) 967 pkt_flags |= (rx_status & NGBE_RXD_ERR_IPCS 968 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD); 969 970 if (rx_status & NGBE_RXD_STAT_L4CS) 971 pkt_flags |= (rx_status & NGBE_RXD_ERR_L4CS 972 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD); 973 974 if (rx_status & NGBE_RXD_STAT_EIPCS && 975 rx_status & NGBE_RXD_ERR_EIPCS) 976 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 977 978 return pkt_flags; 979 } 980 981 /* 982 * LOOK_AHEAD defines how many desc statuses to check beyond the 983 * current descriptor. 984 * It must be a pound define for optimal performance. 985 * Do not change the value of LOOK_AHEAD, as the ngbe_rx_scan_hw_ring 986 * function only works with LOOK_AHEAD=8. 987 */ 988 #define LOOK_AHEAD 8 989 #if (LOOK_AHEAD != 8) 990 #error "PMD NGBE: LOOK_AHEAD must be 8\n" 991 #endif 992 static inline int 993 ngbe_rx_scan_hw_ring(struct ngbe_rx_queue *rxq) 994 { 995 volatile struct ngbe_rx_desc *rxdp; 996 struct ngbe_rx_entry *rxep; 997 struct rte_mbuf *mb; 998 uint16_t pkt_len; 999 uint64_t pkt_flags; 1000 int nb_dd; 1001 uint32_t s[LOOK_AHEAD]; 1002 uint32_t pkt_info[LOOK_AHEAD]; 1003 int i, j, nb_rx = 0; 1004 uint32_t status; 1005 1006 /* get references to current descriptor and S/W ring entry */ 1007 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1008 rxep = &rxq->sw_ring[rxq->rx_tail]; 1009 1010 status = rxdp->qw1.lo.status; 1011 /* check to make sure there is at least 1 packet to receive */ 1012 if (!(status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1013 return 0; 1014 1015 /* 1016 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1017 * reference packets that are ready to be received. 1018 */ 1019 for (i = 0; i < RTE_PMD_NGBE_RX_MAX_BURST; 1020 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1021 /* Read desc statuses backwards to avoid race condition */ 1022 for (j = 0; j < LOOK_AHEAD; j++) 1023 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); 1024 1025 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 1026 1027 /* Compute how many status bits were set */ 1028 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1029 (s[nb_dd] & NGBE_RXD_STAT_DD); nb_dd++) 1030 ; 1031 1032 for (j = 0; j < nb_dd; j++) 1033 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0); 1034 1035 nb_rx += nb_dd; 1036 1037 /* Translate descriptor info to mbuf format */ 1038 for (j = 0; j < nb_dd; ++j) { 1039 mb = rxep[j].mbuf; 1040 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) - 1041 rxq->crc_len; 1042 mb->data_len = pkt_len; 1043 mb->pkt_len = pkt_len; 1044 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag); 1045 1046 /* convert descriptor fields to rte mbuf flags */ 1047 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1048 rxq->vlan_flags); 1049 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]); 1050 pkt_flags |= 1051 ngbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]); 1052 mb->ol_flags = pkt_flags; 1053 mb->packet_type = 1054 ngbe_rxd_pkt_info_to_pkt_type(pkt_info[j], 1055 NGBE_PTID_MASK); 1056 1057 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1058 mb->hash.rss = 1059 rte_le_to_cpu_32(rxdp[j].qw0.dw1); 1060 } 1061 1062 /* Move mbuf pointers from the S/W ring to the stage */ 1063 for (j = 0; j < LOOK_AHEAD; ++j) 1064 rxq->rx_stage[i + j] = rxep[j].mbuf; 1065 1066 /* stop if all requested packets could not be received */ 1067 if (nb_dd != LOOK_AHEAD) 1068 break; 1069 } 1070 1071 /* clear software ring entries so we can cleanup correctly */ 1072 for (i = 0; i < nb_rx; ++i) 1073 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1074 1075 return nb_rx; 1076 } 1077 1078 static inline int 1079 ngbe_rx_alloc_bufs(struct ngbe_rx_queue *rxq, bool reset_mbuf) 1080 { 1081 volatile struct ngbe_rx_desc *rxdp; 1082 struct ngbe_rx_entry *rxep; 1083 struct rte_mbuf *mb; 1084 uint16_t alloc_idx; 1085 __le64 dma_addr; 1086 int diag, i; 1087 1088 /* allocate buffers in bulk directly into the S/W ring */ 1089 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1090 rxep = &rxq->sw_ring[alloc_idx]; 1091 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1092 rxq->rx_free_thresh); 1093 if (unlikely(diag != 0)) 1094 return -ENOMEM; 1095 1096 rxdp = &rxq->rx_ring[alloc_idx]; 1097 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1098 /* populate the static rte mbuf fields */ 1099 mb = rxep[i].mbuf; 1100 if (reset_mbuf) 1101 mb->port = rxq->port_id; 1102 1103 rte_mbuf_refcnt_set(mb, 1); 1104 mb->data_off = RTE_PKTMBUF_HEADROOM; 1105 1106 /* populate the descriptors */ 1107 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1108 NGBE_RXD_HDRADDR(&rxdp[i], 0); 1109 NGBE_RXD_PKTADDR(&rxdp[i], dma_addr); 1110 } 1111 1112 /* update state of internal queue structure */ 1113 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1114 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1115 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1116 1117 /* no errors */ 1118 return 0; 1119 } 1120 1121 static inline uint16_t 1122 ngbe_rx_fill_from_stage(struct ngbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1123 uint16_t nb_pkts) 1124 { 1125 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1126 int i; 1127 1128 /* how many packets are ready to return? */ 1129 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1130 1131 /* copy mbuf pointers to the application's packet list */ 1132 for (i = 0; i < nb_pkts; ++i) 1133 rx_pkts[i] = stage[i]; 1134 1135 /* update internal queue state */ 1136 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1137 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1138 1139 return nb_pkts; 1140 } 1141 1142 static inline uint16_t 1143 ngbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1144 uint16_t nb_pkts) 1145 { 1146 struct ngbe_rx_queue *rxq = (struct ngbe_rx_queue *)rx_queue; 1147 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1148 uint16_t nb_rx = 0; 1149 1150 /* Any previously recv'd pkts will be returned from the Rx stage */ 1151 if (rxq->rx_nb_avail) 1152 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1153 1154 /* Scan the H/W ring for packets to receive */ 1155 nb_rx = (uint16_t)ngbe_rx_scan_hw_ring(rxq); 1156 1157 /* update internal queue state */ 1158 rxq->rx_next_avail = 0; 1159 rxq->rx_nb_avail = nb_rx; 1160 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1161 1162 /* if required, allocate new buffers to replenish descriptors */ 1163 if (rxq->rx_tail > rxq->rx_free_trigger) { 1164 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1165 1166 if (ngbe_rx_alloc_bufs(rxq, true) != 0) { 1167 int i, j; 1168 1169 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1170 "queue_id=%u", (uint16_t)rxq->port_id, 1171 (uint16_t)rxq->queue_id); 1172 1173 dev->data->rx_mbuf_alloc_failed += 1174 rxq->rx_free_thresh; 1175 1176 /* 1177 * Need to rewind any previous receives if we cannot 1178 * allocate new buffers to replenish the old ones. 1179 */ 1180 rxq->rx_nb_avail = 0; 1181 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1182 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1183 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1184 1185 return 0; 1186 } 1187 1188 /* update tail pointer */ 1189 rte_wmb(); 1190 ngbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger); 1191 } 1192 1193 if (rxq->rx_tail >= rxq->nb_rx_desc) 1194 rxq->rx_tail = 0; 1195 1196 /* received any packets this loop? */ 1197 if (rxq->rx_nb_avail) 1198 return ngbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1199 1200 return 0; 1201 } 1202 1203 /* split requests into chunks of size RTE_PMD_NGBE_RX_MAX_BURST */ 1204 uint16_t 1205 ngbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1206 uint16_t nb_pkts) 1207 { 1208 uint16_t nb_rx; 1209 1210 if (unlikely(nb_pkts == 0)) 1211 return 0; 1212 1213 if (likely(nb_pkts <= RTE_PMD_NGBE_RX_MAX_BURST)) 1214 return ngbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1215 1216 /* request is relatively large, chunk it up */ 1217 nb_rx = 0; 1218 while (nb_pkts) { 1219 uint16_t ret, n; 1220 1221 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_NGBE_RX_MAX_BURST); 1222 ret = ngbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1223 nb_rx = (uint16_t)(nb_rx + ret); 1224 nb_pkts = (uint16_t)(nb_pkts - ret); 1225 if (ret < n) 1226 break; 1227 } 1228 1229 return nb_rx; 1230 } 1231 1232 uint16_t 1233 ngbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1234 uint16_t nb_pkts) 1235 { 1236 struct ngbe_rx_queue *rxq; 1237 volatile struct ngbe_rx_desc *rx_ring; 1238 volatile struct ngbe_rx_desc *rxdp; 1239 struct ngbe_rx_entry *sw_ring; 1240 struct ngbe_rx_entry *rxe; 1241 struct rte_mbuf *rxm; 1242 struct rte_mbuf *nmb; 1243 struct ngbe_rx_desc rxd; 1244 uint64_t dma_addr; 1245 uint32_t staterr; 1246 uint32_t pkt_info; 1247 uint16_t pkt_len; 1248 uint16_t rx_id; 1249 uint16_t nb_rx; 1250 uint16_t nb_hold; 1251 uint64_t pkt_flags; 1252 1253 nb_rx = 0; 1254 nb_hold = 0; 1255 rxq = rx_queue; 1256 rx_id = rxq->rx_tail; 1257 rx_ring = rxq->rx_ring; 1258 sw_ring = rxq->sw_ring; 1259 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1260 while (nb_rx < nb_pkts) { 1261 /* 1262 * The order of operations here is important as the DD status 1263 * bit must not be read after any other descriptor fields. 1264 * rx_ring and rxdp are pointing to volatile data so the order 1265 * of accesses cannot be reordered by the compiler. If they were 1266 * not volatile, they could be reordered which could lead to 1267 * using invalid descriptor fields when read from rxd. 1268 */ 1269 rxdp = &rx_ring[rx_id]; 1270 staterr = rxdp->qw1.lo.status; 1271 if (!(staterr & rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) 1272 break; 1273 rxd = *rxdp; 1274 1275 /* 1276 * End of packet. 1277 * 1278 * If the NGBE_RXD_STAT_EOP flag is not set, the Rx packet 1279 * is likely to be invalid and to be dropped by the various 1280 * validation checks performed by the network stack. 1281 * 1282 * Allocate a new mbuf to replenish the RX ring descriptor. 1283 * If the allocation fails: 1284 * - arrange for that Rx descriptor to be the first one 1285 * being parsed the next time the receive function is 1286 * invoked [on the same queue]. 1287 * 1288 * - Stop parsing the Rx ring and return immediately. 1289 * 1290 * This policy do not drop the packet received in the Rx 1291 * descriptor for which the allocation of a new mbuf failed. 1292 * Thus, it allows that packet to be later retrieved if 1293 * mbuf have been freed in the mean time. 1294 * As a side effect, holding Rx descriptors instead of 1295 * systematically giving them back to the NIC may lead to 1296 * Rx ring exhaustion situations. 1297 * However, the NIC can gracefully prevent such situations 1298 * to happen by sending specific "back-pressure" flow control 1299 * frames to its peer(s). 1300 */ 1301 PMD_RX_LOG(DEBUG, 1302 "port_id=%u queue_id=%u rx_id=%u ext_err_stat=0x%08x pkt_len=%u", 1303 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1304 (uint16_t)rx_id, (uint32_t)staterr, 1305 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 1306 1307 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1308 if (nmb == NULL) { 1309 PMD_RX_LOG(DEBUG, 1310 "Rx mbuf alloc failed port_id=%u queue_id=%u", 1311 (uint16_t)rxq->port_id, 1312 (uint16_t)rxq->queue_id); 1313 dev->data->rx_mbuf_alloc_failed++; 1314 break; 1315 } 1316 1317 nb_hold++; 1318 rxe = &sw_ring[rx_id]; 1319 rx_id++; 1320 if (rx_id == rxq->nb_rx_desc) 1321 rx_id = 0; 1322 1323 /* Prefetch next mbuf while processing current one. */ 1324 rte_ngbe_prefetch(sw_ring[rx_id].mbuf); 1325 1326 /* 1327 * When next Rx descriptor is on a cache-line boundary, 1328 * prefetch the next 4 Rx descriptors and the next 8 pointers 1329 * to mbufs. 1330 */ 1331 if ((rx_id & 0x3) == 0) { 1332 rte_ngbe_prefetch(&rx_ring[rx_id]); 1333 rte_ngbe_prefetch(&sw_ring[rx_id]); 1334 } 1335 1336 rxm = rxe->mbuf; 1337 rxe->mbuf = nmb; 1338 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1339 NGBE_RXD_HDRADDR(rxdp, 0); 1340 NGBE_RXD_PKTADDR(rxdp, dma_addr); 1341 1342 /* 1343 * Initialize the returned mbuf. 1344 * 1) setup generic mbuf fields: 1345 * - number of segments, 1346 * - next segment, 1347 * - packet length, 1348 * - Rx port identifier. 1349 * 2) integrate hardware offload data, if any: 1350 * - RSS flag & hash, 1351 * - IP checksum flag, 1352 * - VLAN TCI, if any, 1353 * - error flags. 1354 */ 1355 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - 1356 rxq->crc_len); 1357 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1358 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1359 rxm->nb_segs = 1; 1360 rxm->next = NULL; 1361 rxm->pkt_len = pkt_len; 1362 rxm->data_len = pkt_len; 1363 rxm->port = rxq->port_id; 1364 1365 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); 1366 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ 1367 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); 1368 1369 pkt_flags = rx_desc_status_to_pkt_flags(staterr, 1370 rxq->vlan_flags); 1371 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1372 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1373 rxm->ol_flags = pkt_flags; 1374 rxm->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1375 NGBE_PTID_MASK); 1376 1377 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1378 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); 1379 1380 /* 1381 * Store the mbuf address into the next entry of the array 1382 * of returned packets. 1383 */ 1384 rx_pkts[nb_rx++] = rxm; 1385 } 1386 rxq->rx_tail = rx_id; 1387 1388 /* 1389 * If the number of free Rx descriptors is greater than the Rx free 1390 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1391 * register. 1392 * Update the RDT with the value of the last processed Rx descriptor 1393 * minus 1, to guarantee that the RDT register is never equal to the 1394 * RDH register, which creates a "full" ring situation from the 1395 * hardware point of view... 1396 */ 1397 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1398 if (nb_hold > rxq->rx_free_thresh) { 1399 PMD_RX_LOG(DEBUG, 1400 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 1401 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1402 (uint16_t)rx_id, (uint16_t)nb_hold, 1403 (uint16_t)nb_rx); 1404 rx_id = (uint16_t)((rx_id == 0) ? 1405 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1406 ngbe_set32(rxq->rdt_reg_addr, rx_id); 1407 nb_hold = 0; 1408 } 1409 rxq->nb_rx_hold = nb_hold; 1410 return nb_rx; 1411 } 1412 1413 /** 1414 * ngbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1415 * 1416 * Fill the following info in the HEAD buffer of the Rx cluster: 1417 * - RX port identifier 1418 * - hardware offload data, if any: 1419 * - RSS flag & hash 1420 * - IP checksum flag 1421 * - VLAN TCI, if any 1422 * - error flags 1423 * @head HEAD of the packet cluster 1424 * @desc HW descriptor to get data from 1425 * @rxq Pointer to the Rx queue 1426 */ 1427 static inline void 1428 ngbe_fill_cluster_head_buf(struct rte_mbuf *head, struct ngbe_rx_desc *desc, 1429 struct ngbe_rx_queue *rxq, uint32_t staterr) 1430 { 1431 uint32_t pkt_info; 1432 uint64_t pkt_flags; 1433 1434 head->port = rxq->port_id; 1435 1436 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is 1437 * set in the pkt_flags field. 1438 */ 1439 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag); 1440 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0); 1441 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 1442 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1443 pkt_flags |= ngbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1444 head->ol_flags = pkt_flags; 1445 head->packet_type = ngbe_rxd_pkt_info_to_pkt_type(pkt_info, 1446 NGBE_PTID_MASK); 1447 1448 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1449 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1); 1450 } 1451 1452 /** 1453 * ngbe_recv_pkts_sc - receive handler for scatter case. 1454 * 1455 * @rx_queue Rx queue handle 1456 * @rx_pkts table of received packets 1457 * @nb_pkts size of rx_pkts table 1458 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 1459 * 1460 * Returns the number of received packets/clusters (according to the "bulk 1461 * receive" interface). 1462 */ 1463 static inline uint16_t 1464 ngbe_recv_pkts_sc(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 1465 bool bulk_alloc) 1466 { 1467 struct ngbe_rx_queue *rxq = rx_queue; 1468 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1469 volatile struct ngbe_rx_desc *rx_ring = rxq->rx_ring; 1470 struct ngbe_rx_entry *sw_ring = rxq->sw_ring; 1471 struct ngbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 1472 uint16_t rx_id = rxq->rx_tail; 1473 uint16_t nb_rx = 0; 1474 uint16_t nb_hold = rxq->nb_rx_hold; 1475 uint16_t prev_id = rxq->rx_tail; 1476 1477 while (nb_rx < nb_pkts) { 1478 bool eop; 1479 struct ngbe_rx_entry *rxe; 1480 struct ngbe_scattered_rx_entry *sc_entry; 1481 struct ngbe_scattered_rx_entry *next_sc_entry = NULL; 1482 struct ngbe_rx_entry *next_rxe = NULL; 1483 struct rte_mbuf *first_seg; 1484 struct rte_mbuf *rxm; 1485 struct rte_mbuf *nmb = NULL; 1486 struct ngbe_rx_desc rxd; 1487 uint16_t data_len; 1488 uint16_t next_id; 1489 volatile struct ngbe_rx_desc *rxdp; 1490 uint32_t staterr; 1491 1492 next_desc: 1493 rxdp = &rx_ring[rx_id]; 1494 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); 1495 1496 if (!(staterr & NGBE_RXD_STAT_DD)) 1497 break; 1498 1499 rxd = *rxdp; 1500 1501 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1502 "staterr=0x%x data_len=%u", 1503 rxq->port_id, rxq->queue_id, rx_id, staterr, 1504 rte_le_to_cpu_16(rxd.qw1.hi.len)); 1505 1506 if (!bulk_alloc) { 1507 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1508 if (nmb == NULL) { 1509 PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed " 1510 "port_id=%u queue_id=%u", 1511 rxq->port_id, rxq->queue_id); 1512 1513 dev->data->rx_mbuf_alloc_failed++; 1514 break; 1515 } 1516 } else if (nb_hold > rxq->rx_free_thresh) { 1517 uint16_t next_rdt = rxq->rx_free_trigger; 1518 1519 if (!ngbe_rx_alloc_bufs(rxq, false)) { 1520 rte_wmb(); 1521 ngbe_set32_relaxed(rxq->rdt_reg_addr, 1522 next_rdt); 1523 nb_hold -= rxq->rx_free_thresh; 1524 } else { 1525 PMD_RX_LOG(DEBUG, "Rx bulk alloc failed " 1526 "port_id=%u queue_id=%u", 1527 rxq->port_id, rxq->queue_id); 1528 1529 dev->data->rx_mbuf_alloc_failed++; 1530 break; 1531 } 1532 } 1533 1534 nb_hold++; 1535 rxe = &sw_ring[rx_id]; 1536 eop = staterr & NGBE_RXD_STAT_EOP; 1537 1538 next_id = rx_id + 1; 1539 if (next_id == rxq->nb_rx_desc) 1540 next_id = 0; 1541 1542 /* Prefetch next mbuf while processing current one. */ 1543 rte_ngbe_prefetch(sw_ring[next_id].mbuf); 1544 1545 /* 1546 * When next Rx descriptor is on a cache-line boundary, 1547 * prefetch the next 4 RX descriptors and the next 4 pointers 1548 * to mbufs. 1549 */ 1550 if ((next_id & 0x3) == 0) { 1551 rte_ngbe_prefetch(&rx_ring[next_id]); 1552 rte_ngbe_prefetch(&sw_ring[next_id]); 1553 } 1554 1555 rxm = rxe->mbuf; 1556 1557 if (!bulk_alloc) { 1558 __le64 dma = 1559 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1560 /* 1561 * Update Rx descriptor with the physical address of the 1562 * new data buffer of the new allocated mbuf. 1563 */ 1564 rxe->mbuf = nmb; 1565 1566 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1567 NGBE_RXD_HDRADDR(rxdp, 0); 1568 NGBE_RXD_PKTADDR(rxdp, dma); 1569 } else { 1570 rxe->mbuf = NULL; 1571 } 1572 1573 /* 1574 * Set data length & data buffer address of mbuf. 1575 */ 1576 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len); 1577 rxm->data_len = data_len; 1578 1579 if (!eop) { 1580 uint16_t nextp_id; 1581 1582 nextp_id = next_id; 1583 next_sc_entry = &sw_sc_ring[nextp_id]; 1584 next_rxe = &sw_ring[nextp_id]; 1585 rte_ngbe_prefetch(next_rxe); 1586 } 1587 1588 sc_entry = &sw_sc_ring[rx_id]; 1589 first_seg = sc_entry->fbuf; 1590 sc_entry->fbuf = NULL; 1591 1592 /* 1593 * If this is the first buffer of the received packet, 1594 * set the pointer to the first mbuf of the packet and 1595 * initialize its context. 1596 * Otherwise, update the total length and the number of segments 1597 * of the current scattered packet, and update the pointer to 1598 * the last mbuf of the current packet. 1599 */ 1600 if (first_seg == NULL) { 1601 first_seg = rxm; 1602 first_seg->pkt_len = data_len; 1603 first_seg->nb_segs = 1; 1604 } else { 1605 first_seg->pkt_len += data_len; 1606 first_seg->nb_segs++; 1607 } 1608 1609 prev_id = rx_id; 1610 rx_id = next_id; 1611 1612 /* 1613 * If this is not the last buffer of the received packet, update 1614 * the pointer to the first mbuf at the NEXTP entry in the 1615 * sw_sc_ring and continue to parse the Rx ring. 1616 */ 1617 if (!eop && next_rxe) { 1618 rxm->next = next_rxe->mbuf; 1619 next_sc_entry->fbuf = first_seg; 1620 goto next_desc; 1621 } 1622 1623 /* Initialize the first mbuf of the returned packet */ 1624 ngbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 1625 1626 /* Deal with the case, when HW CRC srip is disabled. */ 1627 first_seg->pkt_len -= rxq->crc_len; 1628 if (unlikely(rxm->data_len <= rxq->crc_len)) { 1629 struct rte_mbuf *lp; 1630 1631 for (lp = first_seg; lp->next != rxm; lp = lp->next) 1632 ; 1633 1634 first_seg->nb_segs--; 1635 lp->data_len -= rxq->crc_len - rxm->data_len; 1636 lp->next = NULL; 1637 rte_pktmbuf_free_seg(rxm); 1638 } else { 1639 rxm->data_len -= rxq->crc_len; 1640 } 1641 1642 /* Prefetch data of first segment, if configured to do so. */ 1643 rte_packet_prefetch((char *)first_seg->buf_addr + 1644 first_seg->data_off); 1645 1646 /* 1647 * Store the mbuf address into the next entry of the array 1648 * of returned packets. 1649 */ 1650 rx_pkts[nb_rx++] = first_seg; 1651 } 1652 1653 /* 1654 * Record index of the next Rx descriptor to probe. 1655 */ 1656 rxq->rx_tail = rx_id; 1657 1658 /* 1659 * If the number of free Rx descriptors is greater than the Rx free 1660 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1661 * register. 1662 * Update the RDT with the value of the last processed Rx descriptor 1663 * minus 1, to guarantee that the RDT register is never equal to the 1664 * RDH register, which creates a "full" ring situation from the 1665 * hardware point of view... 1666 */ 1667 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 1668 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1669 "nb_hold=%u nb_rx=%u", 1670 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 1671 1672 rte_wmb(); 1673 ngbe_set32_relaxed(rxq->rdt_reg_addr, prev_id); 1674 nb_hold = 0; 1675 } 1676 1677 rxq->nb_rx_hold = nb_hold; 1678 return nb_rx; 1679 } 1680 1681 uint16_t 1682 ngbe_recv_pkts_sc_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1683 uint16_t nb_pkts) 1684 { 1685 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, false); 1686 } 1687 1688 uint16_t 1689 ngbe_recv_pkts_sc_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1690 uint16_t nb_pkts) 1691 { 1692 return ngbe_recv_pkts_sc(rx_queue, rx_pkts, nb_pkts, true); 1693 } 1694 1695 /********************************************************************* 1696 * 1697 * Queue management functions 1698 * 1699 **********************************************************************/ 1700 1701 static void 1702 ngbe_tx_queue_release_mbufs(struct ngbe_tx_queue *txq) 1703 { 1704 unsigned int i; 1705 1706 if (txq->sw_ring != NULL) { 1707 for (i = 0; i < txq->nb_tx_desc; i++) { 1708 if (txq->sw_ring[i].mbuf != NULL) { 1709 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 1710 txq->sw_ring[i].mbuf = NULL; 1711 } 1712 } 1713 } 1714 } 1715 1716 static int 1717 ngbe_tx_done_cleanup_full(struct ngbe_tx_queue *txq, uint32_t free_cnt) 1718 { 1719 struct ngbe_tx_entry *swr_ring = txq->sw_ring; 1720 uint16_t i, tx_last, tx_id; 1721 uint16_t nb_tx_free_last; 1722 uint16_t nb_tx_to_clean; 1723 uint32_t pkt_cnt; 1724 1725 /* Start free mbuf from the next of tx_tail */ 1726 tx_last = txq->tx_tail; 1727 tx_id = swr_ring[tx_last].next_id; 1728 1729 if (txq->nb_tx_free == 0 && ngbe_xmit_cleanup(txq)) 1730 return 0; 1731 1732 nb_tx_to_clean = txq->nb_tx_free; 1733 nb_tx_free_last = txq->nb_tx_free; 1734 if (!free_cnt) 1735 free_cnt = txq->nb_tx_desc; 1736 1737 /* Loop through swr_ring to count the amount of 1738 * freeable mubfs and packets. 1739 */ 1740 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 1741 for (i = 0; i < nb_tx_to_clean && 1742 pkt_cnt < free_cnt && 1743 tx_id != tx_last; i++) { 1744 if (swr_ring[tx_id].mbuf != NULL) { 1745 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 1746 swr_ring[tx_id].mbuf = NULL; 1747 1748 /* 1749 * last segment in the packet, 1750 * increment packet count 1751 */ 1752 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 1753 } 1754 1755 tx_id = swr_ring[tx_id].next_id; 1756 } 1757 1758 if (pkt_cnt < free_cnt) { 1759 if (ngbe_xmit_cleanup(txq)) 1760 break; 1761 1762 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 1763 nb_tx_free_last = txq->nb_tx_free; 1764 } 1765 } 1766 1767 return (int)pkt_cnt; 1768 } 1769 1770 static int 1771 ngbe_tx_done_cleanup_simple(struct ngbe_tx_queue *txq, 1772 uint32_t free_cnt) 1773 { 1774 int i, n, cnt; 1775 1776 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 1777 free_cnt = txq->nb_tx_desc; 1778 1779 cnt = free_cnt - free_cnt % txq->tx_free_thresh; 1780 1781 for (i = 0; i < cnt; i += n) { 1782 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh) 1783 break; 1784 1785 n = ngbe_tx_free_bufs(txq); 1786 1787 if (n == 0) 1788 break; 1789 } 1790 1791 return i; 1792 } 1793 1794 int 1795 ngbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 1796 { 1797 struct ngbe_tx_queue *txq = (struct ngbe_tx_queue *)tx_queue; 1798 if (txq->offloads == 0 && 1799 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) 1800 return ngbe_tx_done_cleanup_simple(txq, free_cnt); 1801 1802 return ngbe_tx_done_cleanup_full(txq, free_cnt); 1803 } 1804 1805 static void 1806 ngbe_tx_free_swring(struct ngbe_tx_queue *txq) 1807 { 1808 if (txq != NULL) 1809 rte_free(txq->sw_ring); 1810 } 1811 1812 static void 1813 ngbe_tx_queue_release(struct ngbe_tx_queue *txq) 1814 { 1815 if (txq != NULL) { 1816 if (txq->ops != NULL) { 1817 txq->ops->release_mbufs(txq); 1818 txq->ops->free_swring(txq); 1819 } 1820 rte_free(txq); 1821 } 1822 } 1823 1824 void 1825 ngbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1826 { 1827 ngbe_tx_queue_release(dev->data->tx_queues[qid]); 1828 } 1829 1830 /* (Re)set dynamic ngbe_tx_queue fields to defaults */ 1831 static void 1832 ngbe_reset_tx_queue(struct ngbe_tx_queue *txq) 1833 { 1834 static const struct ngbe_tx_desc zeroed_desc = {0}; 1835 struct ngbe_tx_entry *txe = txq->sw_ring; 1836 uint16_t prev, i; 1837 1838 /* Zero out HW ring memory */ 1839 for (i = 0; i < txq->nb_tx_desc; i++) 1840 txq->tx_ring[i] = zeroed_desc; 1841 1842 /* Initialize SW ring entries */ 1843 prev = (uint16_t)(txq->nb_tx_desc - 1); 1844 for (i = 0; i < txq->nb_tx_desc; i++) { 1845 /* the ring can also be modified by hardware */ 1846 volatile struct ngbe_tx_desc *txd = &txq->tx_ring[i]; 1847 1848 txd->dw3 = rte_cpu_to_le_32(NGBE_TXD_DD); 1849 txe[i].mbuf = NULL; 1850 txe[i].last_id = i; 1851 txe[prev].next_id = i; 1852 prev = i; 1853 } 1854 1855 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 1856 txq->tx_tail = 0; 1857 1858 /* 1859 * Always allow 1 descriptor to be un-allocated to avoid 1860 * a H/W race condition 1861 */ 1862 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 1863 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 1864 txq->ctx_curr = 0; 1865 memset((void *)&txq->ctx_cache, 0, 1866 NGBE_CTX_NUM * sizeof(struct ngbe_ctx_info)); 1867 } 1868 1869 static const struct ngbe_txq_ops def_txq_ops = { 1870 .release_mbufs = ngbe_tx_queue_release_mbufs, 1871 .free_swring = ngbe_tx_free_swring, 1872 .reset = ngbe_reset_tx_queue, 1873 }; 1874 1875 /* Takes an ethdev and a queue and sets up the tx function to be used based on 1876 * the queue parameters. Used in tx_queue_setup by primary process and then 1877 * in dev_init by secondary process when attaching to an existing ethdev. 1878 */ 1879 void 1880 ngbe_set_tx_function(struct rte_eth_dev *dev, struct ngbe_tx_queue *txq) 1881 { 1882 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 1883 if (txq->offloads == 0 && 1884 txq->tx_free_thresh >= RTE_PMD_NGBE_TX_MAX_BURST) { 1885 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 1886 dev->tx_pkt_burst = ngbe_xmit_pkts_simple; 1887 dev->tx_pkt_prepare = NULL; 1888 } else { 1889 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 1890 PMD_INIT_LOG(DEBUG, 1891 " - offloads = 0x%" PRIx64, 1892 txq->offloads); 1893 PMD_INIT_LOG(DEBUG, 1894 " - tx_free_thresh = %lu [RTE_PMD_NGBE_TX_MAX_BURST=%lu]", 1895 (unsigned long)txq->tx_free_thresh, 1896 (unsigned long)RTE_PMD_NGBE_TX_MAX_BURST); 1897 dev->tx_pkt_burst = ngbe_xmit_pkts; 1898 dev->tx_pkt_prepare = ngbe_prep_pkts; 1899 } 1900 } 1901 1902 static const struct { 1903 eth_tx_burst_t pkt_burst; 1904 const char *info; 1905 } ngbe_tx_burst_infos[] = { 1906 { ngbe_xmit_pkts_simple, "Scalar Simple"}, 1907 { ngbe_xmit_pkts, "Scalar"}, 1908 }; 1909 1910 int 1911 ngbe_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 1912 struct rte_eth_burst_mode *mode) 1913 { 1914 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 1915 int ret = -EINVAL; 1916 unsigned int i; 1917 1918 for (i = 0; i < RTE_DIM(ngbe_tx_burst_infos); ++i) { 1919 if (pkt_burst == ngbe_tx_burst_infos[i].pkt_burst) { 1920 snprintf(mode->info, sizeof(mode->info), "%s", 1921 ngbe_tx_burst_infos[i].info); 1922 ret = 0; 1923 break; 1924 } 1925 } 1926 1927 return ret; 1928 } 1929 1930 uint64_t 1931 ngbe_get_tx_port_offloads(struct rte_eth_dev *dev) 1932 { 1933 uint64_t tx_offload_capa; 1934 struct ngbe_hw *hw = ngbe_dev_hw(dev); 1935 1936 tx_offload_capa = 1937 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 1938 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 1939 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 1940 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 1941 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 1942 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | 1943 RTE_ETH_TX_OFFLOAD_TCP_TSO | 1944 RTE_ETH_TX_OFFLOAD_UDP_TSO | 1945 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO | 1946 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | 1947 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | 1948 RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 1949 1950 if (hw->is_pf) 1951 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT; 1952 1953 return tx_offload_capa; 1954 } 1955 1956 int 1957 ngbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 1958 uint16_t queue_idx, 1959 uint16_t nb_desc, 1960 unsigned int socket_id, 1961 const struct rte_eth_txconf *tx_conf) 1962 { 1963 const struct rte_memzone *tz; 1964 struct ngbe_tx_queue *txq; 1965 struct ngbe_hw *hw; 1966 uint16_t tx_free_thresh; 1967 uint64_t offloads; 1968 1969 PMD_INIT_FUNC_TRACE(); 1970 hw = ngbe_dev_hw(dev); 1971 1972 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1973 1974 /* 1975 * The Tx descriptor ring will be cleaned after txq->tx_free_thresh 1976 * descriptors are used or if the number of descriptors required 1977 * to transmit a packet is greater than the number of free Tx 1978 * descriptors. 1979 * One descriptor in the Tx ring is used as a sentinel to avoid a 1980 * H/W race condition, hence the maximum threshold constraints. 1981 * When set to zero use default values. 1982 */ 1983 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 1984 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 1985 if (tx_free_thresh >= (nb_desc - 3)) { 1986 PMD_INIT_LOG(ERR, 1987 "tx_free_thresh must be less than the number of TX descriptors minus 3. (tx_free_thresh=%u port=%d queue=%d)", 1988 (unsigned int)tx_free_thresh, 1989 (int)dev->data->port_id, (int)queue_idx); 1990 return -(EINVAL); 1991 } 1992 1993 if (nb_desc % tx_free_thresh != 0) { 1994 PMD_INIT_LOG(ERR, 1995 "tx_free_thresh must be a divisor of the number of Tx descriptors. (tx_free_thresh=%u port=%d queue=%d)", 1996 (unsigned int)tx_free_thresh, 1997 (int)dev->data->port_id, (int)queue_idx); 1998 return -(EINVAL); 1999 } 2000 2001 /* Free memory prior to re-allocation if needed... */ 2002 if (dev->data->tx_queues[queue_idx] != NULL) { 2003 ngbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 2004 dev->data->tx_queues[queue_idx] = NULL; 2005 } 2006 2007 /* First allocate the Tx queue data structure */ 2008 txq = rte_zmalloc_socket("ethdev Tx queue", 2009 sizeof(struct ngbe_tx_queue), 2010 RTE_CACHE_LINE_SIZE, socket_id); 2011 if (txq == NULL) 2012 return -ENOMEM; 2013 2014 /* 2015 * Allocate Tx ring hardware descriptors. A memzone large enough to 2016 * handle the maximum ring size is allocated in order to allow for 2017 * resizing in later calls to the queue setup function. 2018 */ 2019 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2020 sizeof(struct ngbe_tx_desc) * NGBE_RING_DESC_MAX, 2021 NGBE_ALIGN, socket_id); 2022 if (tz == NULL) { 2023 ngbe_tx_queue_release(txq); 2024 return -ENOMEM; 2025 } 2026 2027 txq->nb_tx_desc = nb_desc; 2028 txq->tx_free_thresh = tx_free_thresh; 2029 txq->pthresh = tx_conf->tx_thresh.pthresh; 2030 txq->hthresh = tx_conf->tx_thresh.hthresh; 2031 txq->wthresh = tx_conf->tx_thresh.wthresh; 2032 txq->queue_id = queue_idx; 2033 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2034 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2035 txq->port_id = dev->data->port_id; 2036 txq->offloads = offloads; 2037 txq->ops = &def_txq_ops; 2038 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2039 2040 txq->tdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXWP(txq->reg_idx)); 2041 txq->tdc_reg_addr = NGBE_REG_ADDR(hw, NGBE_TXCFG(txq->reg_idx)); 2042 2043 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 2044 txq->tx_ring = (struct ngbe_tx_desc *)TMZ_VADDR(tz); 2045 2046 /* Allocate software ring */ 2047 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2048 sizeof(struct ngbe_tx_entry) * nb_desc, 2049 RTE_CACHE_LINE_SIZE, socket_id); 2050 if (txq->sw_ring == NULL) { 2051 ngbe_tx_queue_release(txq); 2052 return -ENOMEM; 2053 } 2054 PMD_INIT_LOG(DEBUG, 2055 "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2056 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2057 2058 /* set up scalar Tx function as appropriate */ 2059 ngbe_set_tx_function(dev, txq); 2060 2061 txq->ops->reset(txq); 2062 2063 dev->data->tx_queues[queue_idx] = txq; 2064 2065 return 0; 2066 } 2067 2068 /** 2069 * ngbe_free_sc_cluster - free the not-yet-completed scattered cluster 2070 * 2071 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2072 * in the sw_sc_ring is not set to NULL but rather points to the next 2073 * mbuf of this RSC aggregation (that has not been completed yet and still 2074 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2075 * will just free first "nb_segs" segments of the cluster explicitly by calling 2076 * an rte_pktmbuf_free_seg(). 2077 * 2078 * @m scattered cluster head 2079 */ 2080 static void 2081 ngbe_free_sc_cluster(struct rte_mbuf *m) 2082 { 2083 uint16_t i, nb_segs = m->nb_segs; 2084 struct rte_mbuf *next_seg; 2085 2086 for (i = 0; i < nb_segs; i++) { 2087 next_seg = m->next; 2088 rte_pktmbuf_free_seg(m); 2089 m = next_seg; 2090 } 2091 } 2092 2093 static void 2094 ngbe_rx_queue_release_mbufs(struct ngbe_rx_queue *rxq) 2095 { 2096 unsigned int i; 2097 2098 if (rxq->sw_ring != NULL) { 2099 for (i = 0; i < rxq->nb_rx_desc; i++) { 2100 if (rxq->sw_ring[i].mbuf != NULL) { 2101 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2102 rxq->sw_ring[i].mbuf = NULL; 2103 } 2104 } 2105 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2106 struct rte_mbuf *mb; 2107 2108 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2109 rte_pktmbuf_free_seg(mb); 2110 } 2111 rxq->rx_nb_avail = 0; 2112 } 2113 2114 if (rxq->sw_sc_ring != NULL) 2115 for (i = 0; i < rxq->nb_rx_desc; i++) 2116 if (rxq->sw_sc_ring[i].fbuf != NULL) { 2117 ngbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2118 rxq->sw_sc_ring[i].fbuf = NULL; 2119 } 2120 } 2121 2122 static void 2123 ngbe_rx_queue_release(struct ngbe_rx_queue *rxq) 2124 { 2125 if (rxq != NULL) { 2126 ngbe_rx_queue_release_mbufs(rxq); 2127 rte_free(rxq->sw_ring); 2128 rte_free(rxq->sw_sc_ring); 2129 rte_free(rxq); 2130 } 2131 } 2132 2133 void 2134 ngbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2135 { 2136 ngbe_rx_queue_release(dev->data->rx_queues[qid]); 2137 } 2138 2139 /* 2140 * Check if Rx Burst Bulk Alloc function can be used. 2141 * Return 2142 * 0: the preconditions are satisfied and the bulk allocation function 2143 * can be used. 2144 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2145 * function must be used. 2146 */ 2147 static inline int 2148 check_rx_burst_bulk_alloc_preconditions(struct ngbe_rx_queue *rxq) 2149 { 2150 int ret = 0; 2151 2152 /* 2153 * Make sure the following pre-conditions are satisfied: 2154 * rxq->rx_free_thresh >= RTE_PMD_NGBE_RX_MAX_BURST 2155 * rxq->rx_free_thresh < rxq->nb_rx_desc 2156 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2157 * Scattered packets are not supported. This should be checked 2158 * outside of this function. 2159 */ 2160 if (rxq->rx_free_thresh < RTE_PMD_NGBE_RX_MAX_BURST) { 2161 PMD_INIT_LOG(DEBUG, 2162 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, RTE_PMD_NGBE_RX_MAX_BURST=%d", 2163 rxq->rx_free_thresh, RTE_PMD_NGBE_RX_MAX_BURST); 2164 ret = -EINVAL; 2165 } else if (rxq->rx_free_thresh >= rxq->nb_rx_desc) { 2166 PMD_INIT_LOG(DEBUG, 2167 "Rx Burst Bulk Alloc Preconditions: rxq->rx_free_thresh=%d, rxq->nb_rx_desc=%d", 2168 rxq->rx_free_thresh, rxq->nb_rx_desc); 2169 ret = -EINVAL; 2170 } else if ((rxq->nb_rx_desc % rxq->rx_free_thresh) != 0) { 2171 PMD_INIT_LOG(DEBUG, 2172 "Rx Burst Bulk Alloc Preconditions: rxq->nb_rx_desc=%d, rxq->rx_free_thresh=%d", 2173 rxq->nb_rx_desc, rxq->rx_free_thresh); 2174 ret = -EINVAL; 2175 } 2176 2177 return ret; 2178 } 2179 2180 /* Reset dynamic ngbe_rx_queue fields back to defaults */ 2181 static void 2182 ngbe_reset_rx_queue(struct ngbe_adapter *adapter, struct ngbe_rx_queue *rxq) 2183 { 2184 static const struct ngbe_rx_desc zeroed_desc = { 2185 {{0}, {0} }, {{0}, {0} } }; 2186 unsigned int i; 2187 uint16_t len = rxq->nb_rx_desc; 2188 2189 /* 2190 * By default, the Rx queue setup function allocates enough memory for 2191 * NGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 2192 * extra memory at the end of the descriptor ring to be zero'd out. 2193 */ 2194 if (adapter->rx_bulk_alloc_allowed) 2195 /* zero out extra memory */ 2196 len += RTE_PMD_NGBE_RX_MAX_BURST; 2197 2198 /* 2199 * Zero out HW ring memory. Zero out extra memory at the end of 2200 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2201 * reads extra memory as zeros. 2202 */ 2203 for (i = 0; i < len; i++) 2204 rxq->rx_ring[i] = zeroed_desc; 2205 2206 /* 2207 * initialize extra software ring entries. Space for these extra 2208 * entries is always allocated 2209 */ 2210 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2211 for (i = rxq->nb_rx_desc; i < len; ++i) 2212 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2213 2214 rxq->rx_nb_avail = 0; 2215 rxq->rx_next_avail = 0; 2216 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2217 rxq->rx_tail = 0; 2218 rxq->nb_rx_hold = 0; 2219 rxq->pkt_first_seg = NULL; 2220 rxq->pkt_last_seg = NULL; 2221 } 2222 2223 uint64_t 2224 ngbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused) 2225 { 2226 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2227 } 2228 2229 uint64_t 2230 ngbe_get_rx_port_offloads(struct rte_eth_dev *dev) 2231 { 2232 uint64_t offloads; 2233 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2234 2235 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 2236 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 2237 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 2238 RTE_ETH_RX_OFFLOAD_KEEP_CRC | 2239 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 2240 RTE_ETH_RX_OFFLOAD_SCATTER; 2241 2242 if (hw->is_pf) 2243 offloads |= (RTE_ETH_RX_OFFLOAD_QINQ_STRIP | 2244 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND); 2245 2246 return offloads; 2247 } 2248 2249 int 2250 ngbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 2251 uint16_t queue_idx, 2252 uint16_t nb_desc, 2253 unsigned int socket_id, 2254 const struct rte_eth_rxconf *rx_conf, 2255 struct rte_mempool *mp) 2256 { 2257 const struct rte_memzone *rz; 2258 struct ngbe_rx_queue *rxq; 2259 struct ngbe_hw *hw; 2260 uint16_t len; 2261 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2262 uint64_t offloads; 2263 2264 PMD_INIT_FUNC_TRACE(); 2265 hw = ngbe_dev_hw(dev); 2266 2267 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2268 2269 /* Free memory prior to re-allocation if needed... */ 2270 if (dev->data->rx_queues[queue_idx] != NULL) { 2271 ngbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 2272 dev->data->rx_queues[queue_idx] = NULL; 2273 } 2274 2275 /* First allocate the Rx queue data structure */ 2276 rxq = rte_zmalloc_socket("ethdev RX queue", 2277 sizeof(struct ngbe_rx_queue), 2278 RTE_CACHE_LINE_SIZE, socket_id); 2279 if (rxq == NULL) 2280 return -ENOMEM; 2281 rxq->mb_pool = mp; 2282 rxq->nb_rx_desc = nb_desc; 2283 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2284 rxq->queue_id = queue_idx; 2285 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2286 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2287 rxq->port_id = dev->data->port_id; 2288 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2289 rxq->crc_len = RTE_ETHER_CRC_LEN; 2290 else 2291 rxq->crc_len = 0; 2292 rxq->drop_en = rx_conf->rx_drop_en; 2293 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2294 rxq->offloads = offloads; 2295 2296 /* 2297 * Allocate Rx ring hardware descriptors. A memzone large enough to 2298 * handle the maximum ring size is allocated in order to allow for 2299 * resizing in later calls to the queue setup function. 2300 */ 2301 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2302 RX_RING_SZ, NGBE_ALIGN, socket_id); 2303 if (rz == NULL) { 2304 ngbe_rx_queue_release(rxq); 2305 return -ENOMEM; 2306 } 2307 2308 /* 2309 * Zero init all the descriptors in the ring. 2310 */ 2311 memset(rz->addr, 0, RX_RING_SZ); 2312 2313 rxq->rdt_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXWP(rxq->reg_idx)); 2314 rxq->rdh_reg_addr = NGBE_REG_ADDR(hw, NGBE_RXRP(rxq->reg_idx)); 2315 2316 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 2317 rxq->rx_ring = (struct ngbe_rx_desc *)TMZ_VADDR(rz); 2318 2319 /* 2320 * Certain constraints must be met in order to use the bulk buffer 2321 * allocation Rx burst function. If any of Rx queues doesn't meet them 2322 * the feature should be disabled for the whole port. 2323 */ 2324 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 2325 PMD_INIT_LOG(DEBUG, 2326 "queue[%d] doesn't meet Rx Bulk Alloc preconditions - canceling the feature for the whole port[%d]", 2327 rxq->queue_id, rxq->port_id); 2328 adapter->rx_bulk_alloc_allowed = false; 2329 } 2330 2331 /* 2332 * Allocate software ring. Allow for space at the end of the 2333 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 2334 * function does not access an invalid memory region. 2335 */ 2336 len = nb_desc; 2337 if (adapter->rx_bulk_alloc_allowed) 2338 len += RTE_PMD_NGBE_RX_MAX_BURST; 2339 2340 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 2341 sizeof(struct ngbe_rx_entry) * len, 2342 RTE_CACHE_LINE_SIZE, socket_id); 2343 if (rxq->sw_ring == NULL) { 2344 ngbe_rx_queue_release(rxq); 2345 return -ENOMEM; 2346 } 2347 2348 /* 2349 * Always allocate even if it's not going to be needed in order to 2350 * simplify the code. 2351 * 2352 * This ring is used in Scattered Rx cases and Scattered Rx may 2353 * be requested in ngbe_dev_rx_init(), which is called later from 2354 * dev_start() flow. 2355 */ 2356 rxq->sw_sc_ring = 2357 rte_zmalloc_socket("rxq->sw_sc_ring", 2358 sizeof(struct ngbe_scattered_rx_entry) * len, 2359 RTE_CACHE_LINE_SIZE, socket_id); 2360 if (rxq->sw_sc_ring == NULL) { 2361 ngbe_rx_queue_release(rxq); 2362 return -ENOMEM; 2363 } 2364 2365 PMD_INIT_LOG(DEBUG, 2366 "sw_ring=%p sw_sc_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2367 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 2368 rxq->rx_ring_phys_addr); 2369 2370 dev->data->rx_queues[queue_idx] = rxq; 2371 2372 ngbe_reset_rx_queue(adapter, rxq); 2373 2374 return 0; 2375 } 2376 2377 uint32_t 2378 ngbe_dev_rx_queue_count(void *rx_queue) 2379 { 2380 #define NGBE_RXQ_SCAN_INTERVAL 4 2381 volatile struct ngbe_rx_desc *rxdp; 2382 struct ngbe_rx_queue *rxq = rx_queue; 2383 uint32_t desc = 0; 2384 2385 rxdp = &rxq->rx_ring[rxq->rx_tail]; 2386 2387 while ((desc < rxq->nb_rx_desc) && 2388 (rxdp->qw1.lo.status & 2389 rte_cpu_to_le_32(NGBE_RXD_STAT_DD))) { 2390 desc += NGBE_RXQ_SCAN_INTERVAL; 2391 rxdp += NGBE_RXQ_SCAN_INTERVAL; 2392 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2393 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2394 desc - rxq->nb_rx_desc]); 2395 } 2396 2397 return desc; 2398 } 2399 2400 int 2401 ngbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2402 { 2403 struct ngbe_rx_queue *rxq = rx_queue; 2404 volatile uint32_t *status; 2405 uint32_t nb_hold, desc; 2406 2407 if (unlikely(offset >= rxq->nb_rx_desc)) 2408 return -EINVAL; 2409 2410 nb_hold = rxq->nb_rx_hold; 2411 if (offset >= rxq->nb_rx_desc - nb_hold) 2412 return RTE_ETH_RX_DESC_UNAVAIL; 2413 2414 desc = rxq->rx_tail + offset; 2415 if (desc >= rxq->nb_rx_desc) 2416 desc -= rxq->nb_rx_desc; 2417 2418 status = &rxq->rx_ring[desc].qw1.lo.status; 2419 if (*status & rte_cpu_to_le_32(NGBE_RXD_STAT_DD)) 2420 return RTE_ETH_RX_DESC_DONE; 2421 2422 return RTE_ETH_RX_DESC_AVAIL; 2423 } 2424 2425 int 2426 ngbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2427 { 2428 struct ngbe_tx_queue *txq = tx_queue; 2429 volatile uint32_t *status; 2430 uint32_t desc; 2431 2432 if (unlikely(offset >= txq->nb_tx_desc)) 2433 return -EINVAL; 2434 2435 desc = txq->tx_tail + offset; 2436 if (desc >= txq->nb_tx_desc) { 2437 desc -= txq->nb_tx_desc; 2438 if (desc >= txq->nb_tx_desc) 2439 desc -= txq->nb_tx_desc; 2440 } 2441 2442 status = &txq->tx_ring[desc].dw3; 2443 if (*status & rte_cpu_to_le_32(NGBE_TXD_DD)) 2444 return RTE_ETH_TX_DESC_DONE; 2445 2446 return RTE_ETH_TX_DESC_FULL; 2447 } 2448 2449 void 2450 ngbe_dev_clear_queues(struct rte_eth_dev *dev) 2451 { 2452 unsigned int i; 2453 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2454 2455 PMD_INIT_FUNC_TRACE(); 2456 2457 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2458 struct ngbe_tx_queue *txq = dev->data->tx_queues[i]; 2459 2460 if (txq != NULL) { 2461 txq->ops->release_mbufs(txq); 2462 txq->ops->reset(txq); 2463 } 2464 } 2465 2466 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2467 struct ngbe_rx_queue *rxq = dev->data->rx_queues[i]; 2468 2469 if (rxq != NULL) { 2470 ngbe_rx_queue_release_mbufs(rxq); 2471 ngbe_reset_rx_queue(adapter, rxq); 2472 } 2473 } 2474 } 2475 2476 void 2477 ngbe_dev_free_queues(struct rte_eth_dev *dev) 2478 { 2479 unsigned int i; 2480 2481 PMD_INIT_FUNC_TRACE(); 2482 2483 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2484 ngbe_dev_rx_queue_release(dev, i); 2485 dev->data->rx_queues[i] = NULL; 2486 } 2487 dev->data->nb_rx_queues = 0; 2488 2489 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2490 ngbe_dev_tx_queue_release(dev, i); 2491 dev->data->tx_queues[i] = NULL; 2492 } 2493 dev->data->nb_tx_queues = 0; 2494 } 2495 2496 /** 2497 * Receive Side Scaling (RSS) 2498 * 2499 * Principles: 2500 * The source and destination IP addresses of the IP header and the source 2501 * and destination ports of TCP/UDP headers, if any, of received packets are 2502 * hashed against a configurable random key to compute a 32-bit RSS hash result. 2503 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 2504 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 2505 * RSS output index which is used as the Rx queue index where to store the 2506 * received packets. 2507 * The following output is supplied in the Rx write-back descriptor: 2508 * - 32-bit result of the Microsoft RSS hash function, 2509 * - 4-bit RSS type field. 2510 */ 2511 2512 /* 2513 * Used as the default key. 2514 */ 2515 static uint8_t rss_intel_key[40] = { 2516 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 2517 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 2518 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 2519 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 2520 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 2521 }; 2522 2523 static void 2524 ngbe_rss_disable(struct rte_eth_dev *dev) 2525 { 2526 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2527 2528 wr32m(hw, NGBE_RACTL, NGBE_RACTL_RSSENA, 0); 2529 } 2530 2531 int 2532 ngbe_dev_rss_hash_update(struct rte_eth_dev *dev, 2533 struct rte_eth_rss_conf *rss_conf) 2534 { 2535 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2536 uint8_t *hash_key; 2537 uint32_t mrqc; 2538 uint32_t rss_key; 2539 uint64_t rss_hf; 2540 uint16_t i; 2541 2542 if (!hw->is_pf) { 2543 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 2544 "NIC."); 2545 return -ENOTSUP; 2546 } 2547 2548 hash_key = rss_conf->rss_key; 2549 if (hash_key) { 2550 /* Fill in RSS hash key */ 2551 for (i = 0; i < 10; i++) { 2552 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); 2553 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); 2554 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); 2555 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); 2556 wr32a(hw, NGBE_REG_RSSKEY, i, rss_key); 2557 } 2558 } 2559 2560 /* Set configured hashing protocols */ 2561 rss_hf = rss_conf->rss_hf & NGBE_RSS_OFFLOAD_ALL; 2562 2563 mrqc = rd32(hw, NGBE_RACTL); 2564 mrqc &= ~NGBE_RACTL_RSSMASK; 2565 if (rss_hf & RTE_ETH_RSS_IPV4) 2566 mrqc |= NGBE_RACTL_RSSIPV4; 2567 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 2568 mrqc |= NGBE_RACTL_RSSIPV4TCP; 2569 if (rss_hf & RTE_ETH_RSS_IPV6 || 2570 rss_hf & RTE_ETH_RSS_IPV6_EX) 2571 mrqc |= NGBE_RACTL_RSSIPV6; 2572 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP || 2573 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 2574 mrqc |= NGBE_RACTL_RSSIPV6TCP; 2575 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 2576 mrqc |= NGBE_RACTL_RSSIPV4UDP; 2577 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP || 2578 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 2579 mrqc |= NGBE_RACTL_RSSIPV6UDP; 2580 2581 if (rss_hf) 2582 mrqc |= NGBE_RACTL_RSSENA; 2583 else 2584 mrqc &= ~NGBE_RACTL_RSSENA; 2585 2586 wr32(hw, NGBE_RACTL, mrqc); 2587 2588 return 0; 2589 } 2590 2591 int 2592 ngbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2593 struct rte_eth_rss_conf *rss_conf) 2594 { 2595 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2596 uint8_t *hash_key; 2597 uint32_t mrqc; 2598 uint32_t rss_key; 2599 uint64_t rss_hf; 2600 uint16_t i; 2601 2602 hash_key = rss_conf->rss_key; 2603 if (hash_key) { 2604 /* Return RSS hash key */ 2605 for (i = 0; i < 10; i++) { 2606 rss_key = rd32a(hw, NGBE_REG_RSSKEY, i); 2607 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); 2608 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); 2609 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); 2610 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); 2611 } 2612 } 2613 2614 rss_hf = 0; 2615 2616 mrqc = rd32(hw, NGBE_RACTL); 2617 if (mrqc & NGBE_RACTL_RSSIPV4) 2618 rss_hf |= RTE_ETH_RSS_IPV4; 2619 if (mrqc & NGBE_RACTL_RSSIPV4TCP) 2620 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 2621 if (mrqc & NGBE_RACTL_RSSIPV6) 2622 rss_hf |= RTE_ETH_RSS_IPV6 | 2623 RTE_ETH_RSS_IPV6_EX; 2624 if (mrqc & NGBE_RACTL_RSSIPV6TCP) 2625 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | 2626 RTE_ETH_RSS_IPV6_TCP_EX; 2627 if (mrqc & NGBE_RACTL_RSSIPV4UDP) 2628 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 2629 if (mrqc & NGBE_RACTL_RSSIPV6UDP) 2630 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | 2631 RTE_ETH_RSS_IPV6_UDP_EX; 2632 if (!(mrqc & NGBE_RACTL_RSSENA)) 2633 rss_hf = 0; 2634 2635 rss_hf &= NGBE_RSS_OFFLOAD_ALL; 2636 2637 rss_conf->rss_hf = rss_hf; 2638 return 0; 2639 } 2640 2641 static void 2642 ngbe_rss_configure(struct rte_eth_dev *dev) 2643 { 2644 struct rte_eth_rss_conf rss_conf; 2645 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2646 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2647 uint32_t reta; 2648 uint16_t i; 2649 uint16_t j; 2650 2651 PMD_INIT_FUNC_TRACE(); 2652 2653 /* 2654 * Fill in redirection table 2655 * The byte-swap is needed because NIC registers are in 2656 * little-endian order. 2657 */ 2658 if (adapter->rss_reta_updated == 0) { 2659 reta = 0; 2660 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { 2661 if (j == dev->data->nb_rx_queues) 2662 j = 0; 2663 reta = (reta >> 8) | LS32(j, 24, 0xFF); 2664 if ((i & 3) == 3) 2665 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta); 2666 } 2667 } 2668 /* 2669 * Configure the RSS key and the RSS protocols used to compute 2670 * the RSS hash of input packets. 2671 */ 2672 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 2673 if (rss_conf.rss_key == NULL) 2674 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 2675 ngbe_dev_rss_hash_update(dev, &rss_conf); 2676 } 2677 2678 void ngbe_configure_port(struct rte_eth_dev *dev) 2679 { 2680 struct ngbe_hw *hw = ngbe_dev_hw(dev); 2681 int i = 0; 2682 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, 2683 0x9100, 0x9200, 2684 0x0000, 0x0000, 2685 0x0000, 0x0000}; 2686 2687 PMD_INIT_FUNC_TRACE(); 2688 2689 /* default outer vlan tpid */ 2690 wr32(hw, NGBE_EXTAG, 2691 NGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | 2692 NGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); 2693 2694 /* default inner vlan tpid */ 2695 wr32m(hw, NGBE_VLANCTL, 2696 NGBE_VLANCTL_TPID_MASK, 2697 NGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); 2698 wr32m(hw, NGBE_DMATXCTRL, 2699 NGBE_DMATXCTRL_TPID_MASK, 2700 NGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); 2701 2702 /* default vlan tpid filters */ 2703 for (i = 0; i < 8; i++) { 2704 wr32m(hw, NGBE_TAGTPID(i / 2), 2705 (i % 2 ? NGBE_TAGTPID_MSB_MASK 2706 : NGBE_TAGTPID_LSB_MASK), 2707 (i % 2 ? NGBE_TAGTPID_MSB(tpids[i]) 2708 : NGBE_TAGTPID_LSB(tpids[i]))); 2709 } 2710 } 2711 2712 static int 2713 ngbe_alloc_rx_queue_mbufs(struct ngbe_rx_queue *rxq) 2714 { 2715 struct ngbe_rx_entry *rxe = rxq->sw_ring; 2716 uint64_t dma_addr; 2717 unsigned int i; 2718 2719 /* Initialize software ring entries */ 2720 for (i = 0; i < rxq->nb_rx_desc; i++) { 2721 /* the ring can also be modified by hardware */ 2722 volatile struct ngbe_rx_desc *rxd; 2723 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 2724 2725 if (mbuf == NULL) { 2726 PMD_INIT_LOG(ERR, "Rx mbuf alloc failed queue_id=%u port_id=%u", 2727 (unsigned int)rxq->queue_id, 2728 (unsigned int)rxq->port_id); 2729 return -ENOMEM; 2730 } 2731 2732 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2733 mbuf->port = rxq->port_id; 2734 2735 dma_addr = 2736 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 2737 rxd = &rxq->rx_ring[i]; 2738 NGBE_RXD_HDRADDR(rxd, 0); 2739 NGBE_RXD_PKTADDR(rxd, dma_addr); 2740 rxe[i].mbuf = mbuf; 2741 } 2742 2743 return 0; 2744 } 2745 2746 static int 2747 ngbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 2748 { 2749 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 2750 switch (dev->data->dev_conf.rxmode.mq_mode) { 2751 case RTE_ETH_MQ_RX_RSS: 2752 ngbe_rss_configure(dev); 2753 break; 2754 2755 case RTE_ETH_MQ_RX_NONE: 2756 default: 2757 /* if mq_mode is none, disable rss mode.*/ 2758 ngbe_rss_disable(dev); 2759 break; 2760 } 2761 } 2762 2763 return 0; 2764 } 2765 2766 void 2767 ngbe_set_rx_function(struct rte_eth_dev *dev) 2768 { 2769 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 2770 2771 if (dev->data->scattered_rx) { 2772 /* 2773 * Set the scattered callback: there are bulk and 2774 * single allocation versions. 2775 */ 2776 if (adapter->rx_bulk_alloc_allowed) { 2777 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 2778 "allocation callback (port=%d).", 2779 dev->data->port_id); 2780 dev->rx_pkt_burst = ngbe_recv_pkts_sc_bulk_alloc; 2781 } else { 2782 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 2783 "single allocation) " 2784 "Scattered Rx callback " 2785 "(port=%d).", 2786 dev->data->port_id); 2787 2788 dev->rx_pkt_burst = ngbe_recv_pkts_sc_single_alloc; 2789 } 2790 /* 2791 * Below we set "simple" callbacks according to port/queues parameters. 2792 * If parameters allow we are going to choose between the following 2793 * callbacks: 2794 * - Bulk Allocation 2795 * - Single buffer allocation (the simplest one) 2796 */ 2797 } else if (adapter->rx_bulk_alloc_allowed) { 2798 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 2799 "satisfied. Rx Burst Bulk Alloc function " 2800 "will be used on port=%d.", 2801 dev->data->port_id); 2802 2803 dev->rx_pkt_burst = ngbe_recv_pkts_bulk_alloc; 2804 } else { 2805 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 2806 "satisfied, or Scattered Rx is requested " 2807 "(port=%d).", 2808 dev->data->port_id); 2809 2810 dev->rx_pkt_burst = ngbe_recv_pkts; 2811 } 2812 } 2813 2814 static const struct { 2815 eth_rx_burst_t pkt_burst; 2816 const char *info; 2817 } ngbe_rx_burst_infos[] = { 2818 { ngbe_recv_pkts_sc_single_alloc, "Scalar Scattered"}, 2819 { ngbe_recv_pkts_sc_bulk_alloc, "Scalar Scattered Bulk Alloc"}, 2820 { ngbe_recv_pkts_bulk_alloc, "Scalar Bulk Alloc"}, 2821 { ngbe_recv_pkts, "Scalar"}, 2822 }; 2823 2824 int 2825 ngbe_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id, 2826 struct rte_eth_burst_mode *mode) 2827 { 2828 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 2829 int ret = -EINVAL; 2830 unsigned int i; 2831 2832 for (i = 0; i < RTE_DIM(ngbe_rx_burst_infos); ++i) { 2833 if (pkt_burst == ngbe_rx_burst_infos[i].pkt_burst) { 2834 snprintf(mode->info, sizeof(mode->info), "%s", 2835 ngbe_rx_burst_infos[i].info); 2836 ret = 0; 2837 break; 2838 } 2839 } 2840 2841 return ret; 2842 } 2843 2844 /* 2845 * Initializes Receive Unit. 2846 */ 2847 int 2848 ngbe_dev_rx_init(struct rte_eth_dev *dev) 2849 { 2850 struct ngbe_hw *hw; 2851 struct ngbe_rx_queue *rxq; 2852 uint64_t bus_addr; 2853 uint32_t fctrl; 2854 uint32_t hlreg0; 2855 uint32_t srrctl; 2856 uint32_t rdrxctl; 2857 uint32_t rxcsum; 2858 uint16_t buf_size; 2859 uint16_t i; 2860 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 2861 2862 PMD_INIT_FUNC_TRACE(); 2863 hw = ngbe_dev_hw(dev); 2864 2865 /* 2866 * Make sure receives are disabled while setting 2867 * up the Rx context (registers, descriptor rings, etc.). 2868 */ 2869 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_ENA, 0); 2870 wr32m(hw, NGBE_PBRXCTL, NGBE_PBRXCTL_ENA, 0); 2871 2872 /* Enable receipt of broadcasted frames */ 2873 fctrl = rd32(hw, NGBE_PSRCTL); 2874 fctrl |= NGBE_PSRCTL_BCA; 2875 wr32(hw, NGBE_PSRCTL, fctrl); 2876 2877 /* 2878 * Configure CRC stripping, if any. 2879 */ 2880 hlreg0 = rd32(hw, NGBE_SECRXCTL); 2881 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2882 hlreg0 &= ~NGBE_SECRXCTL_CRCSTRIP; 2883 else 2884 hlreg0 |= NGBE_SECRXCTL_CRCSTRIP; 2885 hlreg0 &= ~NGBE_SECRXCTL_XDSA; 2886 wr32(hw, NGBE_SECRXCTL, hlreg0); 2887 2888 /* 2889 * Configure jumbo frame support, if any. 2890 */ 2891 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK, 2892 NGBE_FRMSZ_MAX(dev->data->mtu + NGBE_ETH_OVERHEAD)); 2893 2894 /* 2895 * If loopback mode is configured, set LPBK bit. 2896 */ 2897 hlreg0 = rd32(hw, NGBE_PSRCTL); 2898 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 2899 hlreg0 |= NGBE_PSRCTL_LBENA; 2900 else 2901 hlreg0 &= ~NGBE_PSRCTL_LBENA; 2902 2903 wr32(hw, NGBE_PSRCTL, hlreg0); 2904 2905 /* 2906 * Assume no header split and no VLAN strip support 2907 * on any Rx queue first . 2908 */ 2909 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2910 2911 /* Setup Rx queues */ 2912 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2913 rxq = dev->data->rx_queues[i]; 2914 2915 /* 2916 * Reset crc_len in case it was changed after queue setup by a 2917 * call to configure. 2918 */ 2919 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2920 rxq->crc_len = RTE_ETHER_CRC_LEN; 2921 else 2922 rxq->crc_len = 0; 2923 2924 /* Setup the Base and Length of the Rx Descriptor Rings */ 2925 bus_addr = rxq->rx_ring_phys_addr; 2926 wr32(hw, NGBE_RXBAL(rxq->reg_idx), 2927 (uint32_t)(bus_addr & BIT_MASK32)); 2928 wr32(hw, NGBE_RXBAH(rxq->reg_idx), 2929 (uint32_t)(bus_addr >> 32)); 2930 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 2931 wr32(hw, NGBE_RXWP(rxq->reg_idx), 0); 2932 2933 srrctl = NGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 2934 2935 /* Set if packets are dropped when no descriptors available */ 2936 if (rxq->drop_en) 2937 srrctl |= NGBE_RXCFG_DROP; 2938 2939 /* 2940 * Configure the Rx buffer size in the PKTLEN field of 2941 * the RXCFG register of the queue. 2942 * The value is in 1 KB resolution. Valid values can be from 2943 * 1 KB to 16 KB. 2944 */ 2945 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 2946 RTE_PKTMBUF_HEADROOM); 2947 buf_size = ROUND_DOWN(buf_size, 0x1 << 10); 2948 srrctl |= NGBE_RXCFG_PKTLEN(buf_size); 2949 2950 wr32(hw, NGBE_RXCFG(rxq->reg_idx), srrctl); 2951 2952 /* It adds dual VLAN length for supporting dual VLAN */ 2953 if (dev->data->mtu + NGBE_ETH_OVERHEAD + 2954 2 * RTE_VLAN_HLEN > buf_size) 2955 dev->data->scattered_rx = 1; 2956 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 2957 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 2958 } 2959 2960 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 2961 dev->data->scattered_rx = 1; 2962 2963 /* 2964 * Device configured with multiple RX queues. 2965 */ 2966 ngbe_dev_mq_rx_configure(dev); 2967 2968 /* 2969 * Setup the Checksum Register. 2970 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 2971 * Enable IP/L4 checksum computation by hardware if requested to do so. 2972 */ 2973 rxcsum = rd32(hw, NGBE_PSRCTL); 2974 rxcsum |= NGBE_PSRCTL_PCSD; 2975 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 2976 rxcsum |= NGBE_PSRCTL_L4CSUM; 2977 else 2978 rxcsum &= ~NGBE_PSRCTL_L4CSUM; 2979 2980 wr32(hw, NGBE_PSRCTL, rxcsum); 2981 2982 if (hw->is_pf) { 2983 rdrxctl = rd32(hw, NGBE_SECRXCTL); 2984 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2985 rdrxctl &= ~NGBE_SECRXCTL_CRCSTRIP; 2986 else 2987 rdrxctl |= NGBE_SECRXCTL_CRCSTRIP; 2988 wr32(hw, NGBE_SECRXCTL, rdrxctl); 2989 } 2990 2991 ngbe_set_rx_function(dev); 2992 2993 return 0; 2994 } 2995 2996 /* 2997 * Initializes Transmit Unit. 2998 */ 2999 void 3000 ngbe_dev_tx_init(struct rte_eth_dev *dev) 3001 { 3002 struct ngbe_hw *hw; 3003 struct ngbe_tx_queue *txq; 3004 uint64_t bus_addr; 3005 uint16_t i; 3006 3007 PMD_INIT_FUNC_TRACE(); 3008 hw = ngbe_dev_hw(dev); 3009 3010 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_ODSA, NGBE_SECTXCTL_ODSA); 3011 wr32m(hw, NGBE_SECTXCTL, NGBE_SECTXCTL_XDSA, 0); 3012 3013 /* Setup the Base and Length of the Tx Descriptor Rings */ 3014 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3015 txq = dev->data->tx_queues[i]; 3016 3017 bus_addr = txq->tx_ring_phys_addr; 3018 wr32(hw, NGBE_TXBAL(txq->reg_idx), 3019 (uint32_t)(bus_addr & BIT_MASK32)); 3020 wr32(hw, NGBE_TXBAH(txq->reg_idx), 3021 (uint32_t)(bus_addr >> 32)); 3022 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_BUFLEN_MASK, 3023 NGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 3024 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 3025 wr32(hw, NGBE_TXRP(txq->reg_idx), 0); 3026 wr32(hw, NGBE_TXWP(txq->reg_idx), 0); 3027 } 3028 } 3029 3030 /* 3031 * Set up link loopback mode Tx->Rx. 3032 */ 3033 static inline void 3034 ngbe_setup_loopback_link(struct ngbe_hw *hw) 3035 { 3036 PMD_INIT_FUNC_TRACE(); 3037 3038 wr32m(hw, NGBE_MACRXCFG, NGBE_MACRXCFG_LB, NGBE_MACRXCFG_LB); 3039 3040 msec_delay(50); 3041 } 3042 3043 /* 3044 * Start Transmit and Receive Units. 3045 */ 3046 int 3047 ngbe_dev_rxtx_start(struct rte_eth_dev *dev) 3048 { 3049 struct ngbe_hw *hw; 3050 struct ngbe_tx_queue *txq; 3051 struct ngbe_rx_queue *rxq; 3052 uint32_t dmatxctl; 3053 uint32_t rxctrl; 3054 uint16_t i; 3055 int ret = 0; 3056 3057 PMD_INIT_FUNC_TRACE(); 3058 hw = ngbe_dev_hw(dev); 3059 3060 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3061 txq = dev->data->tx_queues[i]; 3062 /* Setup Transmit Threshold Registers */ 3063 wr32m(hw, NGBE_TXCFG(txq->reg_idx), 3064 NGBE_TXCFG_HTHRESH_MASK | 3065 NGBE_TXCFG_WTHRESH_MASK, 3066 NGBE_TXCFG_HTHRESH(txq->hthresh) | 3067 NGBE_TXCFG_WTHRESH(txq->wthresh)); 3068 } 3069 3070 dmatxctl = rd32(hw, NGBE_DMATXCTRL); 3071 dmatxctl |= NGBE_DMATXCTRL_ENA; 3072 wr32(hw, NGBE_DMATXCTRL, dmatxctl); 3073 3074 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3075 txq = dev->data->tx_queues[i]; 3076 if (txq->tx_deferred_start == 0) { 3077 ret = ngbe_dev_tx_queue_start(dev, i); 3078 if (ret < 0) 3079 return ret; 3080 } 3081 } 3082 3083 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3084 rxq = dev->data->rx_queues[i]; 3085 if (rxq->rx_deferred_start == 0) { 3086 ret = ngbe_dev_rx_queue_start(dev, i); 3087 if (ret < 0) 3088 return ret; 3089 } 3090 } 3091 3092 /* Enable Receive engine */ 3093 rxctrl = rd32(hw, NGBE_PBRXCTL); 3094 rxctrl |= NGBE_PBRXCTL_ENA; 3095 hw->mac.enable_rx_dma(hw, rxctrl); 3096 3097 /* If loopback mode is enabled, set up the link accordingly */ 3098 if (hw->is_pf && dev->data->dev_conf.lpbk_mode) 3099 ngbe_setup_loopback_link(hw); 3100 3101 return 0; 3102 } 3103 3104 void 3105 ngbe_dev_save_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 3106 { 3107 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 3108 *(reg++) = rd32(hw, NGBE_RXBAL(rx_queue_id)); 3109 *(reg++) = rd32(hw, NGBE_RXBAH(rx_queue_id)); 3110 *(reg++) = rd32(hw, NGBE_RXCFG(rx_queue_id)); 3111 } 3112 3113 void 3114 ngbe_dev_store_rx_queue(struct ngbe_hw *hw, uint16_t rx_queue_id) 3115 { 3116 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 3117 wr32(hw, NGBE_RXBAL(rx_queue_id), *(reg++)); 3118 wr32(hw, NGBE_RXBAH(rx_queue_id), *(reg++)); 3119 wr32(hw, NGBE_RXCFG(rx_queue_id), *(reg++) & ~NGBE_RXCFG_ENA); 3120 } 3121 3122 void 3123 ngbe_dev_save_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 3124 { 3125 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 3126 *(reg++) = rd32(hw, NGBE_TXBAL(tx_queue_id)); 3127 *(reg++) = rd32(hw, NGBE_TXBAH(tx_queue_id)); 3128 *(reg++) = rd32(hw, NGBE_TXCFG(tx_queue_id)); 3129 } 3130 3131 void 3132 ngbe_dev_store_tx_queue(struct ngbe_hw *hw, uint16_t tx_queue_id) 3133 { 3134 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 3135 wr32(hw, NGBE_TXBAL(tx_queue_id), *(reg++)); 3136 wr32(hw, NGBE_TXBAH(tx_queue_id), *(reg++)); 3137 wr32(hw, NGBE_TXCFG(tx_queue_id), *(reg++) & ~NGBE_TXCFG_ENA); 3138 } 3139 3140 /* 3141 * Start Receive Units for specified queue. 3142 */ 3143 int 3144 ngbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3145 { 3146 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3147 struct ngbe_rx_queue *rxq; 3148 uint32_t rxdctl; 3149 int poll_ms; 3150 3151 PMD_INIT_FUNC_TRACE(); 3152 3153 rxq = dev->data->rx_queues[rx_queue_id]; 3154 3155 /* Allocate buffers for descriptor rings */ 3156 if (ngbe_alloc_rx_queue_mbufs(rxq) != 0) { 3157 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 3158 rx_queue_id); 3159 return -1; 3160 } 3161 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3162 rxdctl |= NGBE_RXCFG_ENA; 3163 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxdctl); 3164 3165 /* Wait until Rx Enable ready */ 3166 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3167 do { 3168 rte_delay_ms(1); 3169 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3170 } while (--poll_ms && !(rxdctl & NGBE_RXCFG_ENA)); 3171 if (poll_ms == 0) 3172 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 3173 rte_wmb(); 3174 wr32(hw, NGBE_RXRP(rxq->reg_idx), 0); 3175 wr32(hw, NGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 3176 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3177 3178 return 0; 3179 } 3180 3181 /* 3182 * Stop Receive Units for specified queue. 3183 */ 3184 int 3185 ngbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3186 { 3187 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3188 struct ngbe_adapter *adapter = ngbe_dev_adapter(dev); 3189 struct ngbe_rx_queue *rxq; 3190 uint32_t rxdctl; 3191 int poll_ms; 3192 3193 PMD_INIT_FUNC_TRACE(); 3194 3195 rxq = dev->data->rx_queues[rx_queue_id]; 3196 3197 ngbe_dev_save_rx_queue(hw, rxq->reg_idx); 3198 wr32m(hw, NGBE_RXCFG(rxq->reg_idx), NGBE_RXCFG_ENA, 0); 3199 3200 /* Wait until Rx Enable bit clear */ 3201 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3202 do { 3203 rte_delay_ms(1); 3204 rxdctl = rd32(hw, NGBE_RXCFG(rxq->reg_idx)); 3205 } while (--poll_ms && (rxdctl & NGBE_RXCFG_ENA)); 3206 if (poll_ms == 0) 3207 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 3208 3209 rte_delay_us(RTE_NGBE_WAIT_100_US); 3210 ngbe_dev_store_rx_queue(hw, rxq->reg_idx); 3211 3212 ngbe_rx_queue_release_mbufs(rxq); 3213 ngbe_reset_rx_queue(adapter, rxq); 3214 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3215 3216 return 0; 3217 } 3218 3219 /* 3220 * Start Transmit Units for specified queue. 3221 */ 3222 int 3223 ngbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3224 { 3225 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3226 struct ngbe_tx_queue *txq; 3227 uint32_t txdctl; 3228 int poll_ms; 3229 3230 PMD_INIT_FUNC_TRACE(); 3231 3232 txq = dev->data->tx_queues[tx_queue_id]; 3233 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, NGBE_TXCFG_ENA); 3234 3235 /* Wait until Tx Enable ready */ 3236 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3237 do { 3238 rte_delay_ms(1); 3239 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3240 } while (--poll_ms && !(txdctl & NGBE_TXCFG_ENA)); 3241 if (poll_ms == 0) 3242 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 3243 tx_queue_id); 3244 3245 rte_wmb(); 3246 wr32(hw, NGBE_TXWP(txq->reg_idx), txq->tx_tail); 3247 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 3248 3249 return 0; 3250 } 3251 3252 /* 3253 * Stop Transmit Units for specified queue. 3254 */ 3255 int 3256 ngbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 3257 { 3258 struct ngbe_hw *hw = ngbe_dev_hw(dev); 3259 struct ngbe_tx_queue *txq; 3260 uint32_t txdctl; 3261 uint32_t txtdh, txtdt; 3262 int poll_ms; 3263 3264 PMD_INIT_FUNC_TRACE(); 3265 3266 txq = dev->data->tx_queues[tx_queue_id]; 3267 3268 /* Wait until Tx queue is empty */ 3269 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3270 do { 3271 rte_delay_us(RTE_NGBE_WAIT_100_US); 3272 txtdh = rd32(hw, NGBE_TXRP(txq->reg_idx)); 3273 txtdt = rd32(hw, NGBE_TXWP(txq->reg_idx)); 3274 } while (--poll_ms && (txtdh != txtdt)); 3275 if (poll_ms == 0) 3276 PMD_INIT_LOG(ERR, "Tx Queue %d is not empty when stopping.", 3277 tx_queue_id); 3278 3279 ngbe_dev_save_tx_queue(hw, txq->reg_idx); 3280 wr32m(hw, NGBE_TXCFG(txq->reg_idx), NGBE_TXCFG_ENA, 0); 3281 3282 /* Wait until Tx Enable bit clear */ 3283 poll_ms = RTE_NGBE_REGISTER_POLL_WAIT_10_MS; 3284 do { 3285 rte_delay_ms(1); 3286 txdctl = rd32(hw, NGBE_TXCFG(txq->reg_idx)); 3287 } while (--poll_ms && (txdctl & NGBE_TXCFG_ENA)); 3288 if (poll_ms == 0) 3289 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 3290 tx_queue_id); 3291 3292 rte_delay_us(RTE_NGBE_WAIT_100_US); 3293 ngbe_dev_store_tx_queue(hw, txq->reg_idx); 3294 3295 if (txq->ops != NULL) { 3296 txq->ops->release_mbufs(txq); 3297 txq->ops->reset(txq); 3298 } 3299 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 3300 3301 return 0; 3302 } 3303 3304 void 3305 ngbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3306 struct rte_eth_rxq_info *qinfo) 3307 { 3308 struct ngbe_rx_queue *rxq; 3309 3310 rxq = dev->data->rx_queues[queue_id]; 3311 3312 qinfo->mp = rxq->mb_pool; 3313 qinfo->scattered_rx = dev->data->scattered_rx; 3314 qinfo->nb_desc = rxq->nb_rx_desc; 3315 3316 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 3317 qinfo->conf.rx_drop_en = rxq->drop_en; 3318 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 3319 qinfo->conf.offloads = rxq->offloads; 3320 } 3321 3322 void 3323 ngbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 3324 struct rte_eth_txq_info *qinfo) 3325 { 3326 struct ngbe_tx_queue *txq; 3327 3328 txq = dev->data->tx_queues[queue_id]; 3329 3330 qinfo->nb_desc = txq->nb_tx_desc; 3331 3332 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 3333 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 3334 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 3335 3336 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 3337 qinfo->conf.offloads = txq->offloads; 3338 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 3339 } 3340