1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 * Copyright(c) 2018 Synopsys, Inc. All rights reserved. 4 */ 5 6 #include "axgbe_ethdev.h" 7 #include "axgbe_rxtx.h" 8 #include "axgbe_phy.h" 9 10 #include <rte_time.h> 11 #include <rte_mempool.h> 12 #include <rte_mbuf.h> 13 #include <rte_vect.h> 14 15 static void 16 axgbe_rx_queue_release(struct axgbe_rx_queue *rx_queue) 17 { 18 uint16_t i; 19 struct rte_mbuf **sw_ring; 20 21 if (rx_queue) { 22 sw_ring = rx_queue->sw_ring; 23 if (sw_ring) { 24 for (i = 0; i < rx_queue->nb_desc; i++) { 25 if (sw_ring[i]) 26 rte_pktmbuf_free(sw_ring[i]); 27 } 28 rte_free(sw_ring); 29 } 30 rte_free(rx_queue); 31 } 32 } 33 34 void axgbe_dev_rx_queue_release(void *rxq) 35 { 36 axgbe_rx_queue_release(rxq); 37 } 38 39 int axgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 40 uint16_t nb_desc, unsigned int socket_id, 41 const struct rte_eth_rxconf *rx_conf, 42 struct rte_mempool *mp) 43 { 44 PMD_INIT_FUNC_TRACE(); 45 uint32_t size; 46 const struct rte_memzone *dma; 47 struct axgbe_rx_queue *rxq; 48 uint32_t rx_desc = nb_desc; 49 struct axgbe_port *pdata = dev->data->dev_private; 50 51 /* 52 * validate Rx descriptors count 53 * should be power of 2 and less than h/w supported 54 */ 55 if ((!rte_is_power_of_2(rx_desc)) || 56 rx_desc > pdata->rx_desc_count) 57 return -EINVAL; 58 /* First allocate the rx queue data structure */ 59 rxq = rte_zmalloc_socket("ethdev RX queue", 60 sizeof(struct axgbe_rx_queue), 61 RTE_CACHE_LINE_SIZE, socket_id); 62 if (!rxq) { 63 PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!"); 64 return -ENOMEM; 65 } 66 67 rxq->cur = 0; 68 rxq->dirty = 0; 69 rxq->pdata = pdata; 70 rxq->mb_pool = mp; 71 rxq->queue_id = queue_idx; 72 rxq->port_id = dev->data->port_id; 73 rxq->nb_desc = rx_desc; 74 rxq->dma_regs = (void *)((uint8_t *)pdata->xgmac_regs + DMA_CH_BASE + 75 (DMA_CH_INC * rxq->queue_id)); 76 rxq->dma_tail_reg = (volatile uint32_t *)((uint8_t *)rxq->dma_regs + 77 DMA_CH_RDTR_LO); 78 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) 79 rxq->crc_len = RTE_ETHER_CRC_LEN; 80 else 81 rxq->crc_len = 0; 82 83 /* CRC strip in AXGBE supports per port not per queue */ 84 pdata->crc_strip_enable = (rxq->crc_len == 0) ? 1 : 0; 85 rxq->free_thresh = rx_conf->rx_free_thresh ? 86 rx_conf->rx_free_thresh : AXGBE_RX_FREE_THRESH; 87 if (rxq->free_thresh > rxq->nb_desc) 88 rxq->free_thresh = rxq->nb_desc >> 3; 89 90 /* Allocate RX ring hardware descriptors */ 91 size = rxq->nb_desc * sizeof(union axgbe_rx_desc); 92 dma = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, size, 128, 93 socket_id); 94 if (!dma) { 95 PMD_DRV_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed\n"); 96 axgbe_rx_queue_release(rxq); 97 return -ENOMEM; 98 } 99 rxq->ring_phys_addr = (uint64_t)dma->iova; 100 rxq->desc = (volatile union axgbe_rx_desc *)dma->addr; 101 memset((void *)rxq->desc, 0, size); 102 /* Allocate software ring */ 103 size = rxq->nb_desc * sizeof(struct rte_mbuf *); 104 rxq->sw_ring = rte_zmalloc_socket("sw_ring", size, 105 RTE_CACHE_LINE_SIZE, 106 socket_id); 107 if (!rxq->sw_ring) { 108 PMD_DRV_LOG(ERR, "rte_zmalloc for sw_ring failed\n"); 109 axgbe_rx_queue_release(rxq); 110 return -ENOMEM; 111 } 112 dev->data->rx_queues[queue_idx] = rxq; 113 if (!pdata->rx_queues) 114 pdata->rx_queues = dev->data->rx_queues; 115 116 return 0; 117 } 118 119 static void axgbe_prepare_rx_stop(struct axgbe_port *pdata, 120 unsigned int queue) 121 { 122 unsigned int rx_status; 123 unsigned long rx_timeout; 124 125 /* The Rx engine cannot be stopped if it is actively processing 126 * packets. Wait for the Rx queue to empty the Rx fifo. Don't 127 * wait forever though... 128 */ 129 rx_timeout = rte_get_timer_cycles() + (AXGBE_DMA_STOP_TIMEOUT * 130 rte_get_timer_hz()); 131 132 while (time_before(rte_get_timer_cycles(), rx_timeout)) { 133 rx_status = AXGMAC_MTL_IOREAD(pdata, queue, MTL_Q_RQDR); 134 if ((AXGMAC_GET_BITS(rx_status, MTL_Q_RQDR, PRXQ) == 0) && 135 (AXGMAC_GET_BITS(rx_status, MTL_Q_RQDR, RXQSTS) == 0)) 136 break; 137 138 rte_delay_us(900); 139 } 140 141 if (!time_before(rte_get_timer_cycles(), rx_timeout)) 142 PMD_DRV_LOG(ERR, 143 "timed out waiting for Rx queue %u to empty\n", 144 queue); 145 } 146 147 void axgbe_dev_disable_rx(struct rte_eth_dev *dev) 148 { 149 struct axgbe_rx_queue *rxq; 150 struct axgbe_port *pdata = dev->data->dev_private; 151 unsigned int i; 152 153 /* Disable MAC Rx */ 154 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 0); 155 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, CST, 0); 156 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, ACS, 0); 157 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, RE, 0); 158 159 /* Prepare for Rx DMA channel stop */ 160 for (i = 0; i < dev->data->nb_rx_queues; i++) { 161 rxq = dev->data->rx_queues[i]; 162 axgbe_prepare_rx_stop(pdata, i); 163 } 164 /* Disable each Rx queue */ 165 AXGMAC_IOWRITE(pdata, MAC_RQC0R, 0); 166 for (i = 0; i < dev->data->nb_rx_queues; i++) { 167 rxq = dev->data->rx_queues[i]; 168 /* Disable Rx DMA channel */ 169 AXGMAC_DMA_IOWRITE_BITS(rxq, DMA_CH_RCR, SR, 0); 170 } 171 } 172 173 void axgbe_dev_enable_rx(struct rte_eth_dev *dev) 174 { 175 struct axgbe_rx_queue *rxq; 176 struct axgbe_port *pdata = dev->data->dev_private; 177 unsigned int i; 178 unsigned int reg_val = 0; 179 180 for (i = 0; i < dev->data->nb_rx_queues; i++) { 181 rxq = dev->data->rx_queues[i]; 182 /* Enable Rx DMA channel */ 183 AXGMAC_DMA_IOWRITE_BITS(rxq, DMA_CH_RCR, SR, 1); 184 } 185 186 reg_val = 0; 187 for (i = 0; i < pdata->rx_q_count; i++) 188 reg_val |= (0x02 << (i << 1)); 189 AXGMAC_IOWRITE(pdata, MAC_RQC0R, reg_val); 190 191 /* Enable MAC Rx */ 192 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 1); 193 /* Frame is forwarded after stripping CRC to application*/ 194 if (pdata->crc_strip_enable) { 195 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, CST, 1); 196 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, ACS, 1); 197 } 198 AXGMAC_IOWRITE_BITS(pdata, MAC_RCR, RE, 1); 199 } 200 201 /* Rx function one to one refresh */ 202 uint16_t 203 axgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 204 uint16_t nb_pkts) 205 { 206 PMD_INIT_FUNC_TRACE(); 207 uint16_t nb_rx = 0; 208 struct axgbe_rx_queue *rxq = rx_queue; 209 volatile union axgbe_rx_desc *desc; 210 uint64_t old_dirty = rxq->dirty; 211 struct rte_mbuf *mbuf, *tmbuf; 212 unsigned int err; 213 uint32_t error_status; 214 uint16_t idx, pidx, pkt_len; 215 216 idx = AXGBE_GET_DESC_IDX(rxq, rxq->cur); 217 while (nb_rx < nb_pkts) { 218 if (unlikely(idx == rxq->nb_desc)) 219 idx = 0; 220 221 desc = &rxq->desc[idx]; 222 223 if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, OWN)) 224 break; 225 tmbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 226 if (unlikely(!tmbuf)) { 227 PMD_DRV_LOG(ERR, "RX mbuf alloc failed port_id = %u" 228 " queue_id = %u\n", 229 (unsigned int)rxq->port_id, 230 (unsigned int)rxq->queue_id); 231 rte_eth_devices[ 232 rxq->port_id].data->rx_mbuf_alloc_failed++; 233 rxq->rx_mbuf_alloc_failed++; 234 break; 235 } 236 pidx = idx + 1; 237 if (unlikely(pidx == rxq->nb_desc)) 238 pidx = 0; 239 240 rte_prefetch0(rxq->sw_ring[pidx]); 241 if ((pidx & 0x3) == 0) { 242 rte_prefetch0(&rxq->desc[pidx]); 243 rte_prefetch0(&rxq->sw_ring[pidx]); 244 } 245 246 mbuf = rxq->sw_ring[idx]; 247 /* Check for any errors and free mbuf*/ 248 err = AXGMAC_GET_BITS_LE(desc->write.desc3, 249 RX_NORMAL_DESC3, ES); 250 error_status = 0; 251 if (unlikely(err)) { 252 error_status = desc->write.desc3 & AXGBE_ERR_STATUS; 253 if ((error_status != AXGBE_L3_CSUM_ERR) && 254 (error_status != AXGBE_L4_CSUM_ERR)) { 255 rxq->errors++; 256 rte_pktmbuf_free(mbuf); 257 goto err_set; 258 } 259 } 260 if (rxq->pdata->rx_csum_enable) { 261 mbuf->ol_flags = 0; 262 mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 263 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 264 if (unlikely(error_status == AXGBE_L3_CSUM_ERR)) { 265 mbuf->ol_flags &= ~PKT_RX_IP_CKSUM_GOOD; 266 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 267 mbuf->ol_flags &= ~PKT_RX_L4_CKSUM_GOOD; 268 mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 269 } else if ( 270 unlikely(error_status == AXGBE_L4_CSUM_ERR)) { 271 mbuf->ol_flags &= ~PKT_RX_L4_CKSUM_GOOD; 272 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 273 } 274 } 275 rte_prefetch1(rte_pktmbuf_mtod(mbuf, void *)); 276 /* Get the RSS hash */ 277 if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, RSV)) 278 mbuf->hash.rss = rte_le_to_cpu_32(desc->write.desc1); 279 /* Indicate if a Context Descriptor is next */ 280 if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, CDA)) 281 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP 282 | PKT_RX_IEEE1588_TMST; 283 pkt_len = AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, 284 PL) - rxq->crc_len; 285 /* Mbuf populate */ 286 mbuf->next = NULL; 287 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 288 mbuf->nb_segs = 1; 289 mbuf->port = rxq->port_id; 290 mbuf->pkt_len = pkt_len; 291 mbuf->data_len = pkt_len; 292 rxq->bytes += pkt_len; 293 rx_pkts[nb_rx++] = mbuf; 294 err_set: 295 rxq->cur++; 296 rxq->sw_ring[idx++] = tmbuf; 297 desc->read.baddr = 298 rte_cpu_to_le_64(rte_mbuf_data_iova_default(tmbuf)); 299 memset((void *)(&desc->read.desc2), 0, 8); 300 AXGMAC_SET_BITS_LE(desc->read.desc3, RX_NORMAL_DESC3, OWN, 1); 301 rxq->dirty++; 302 } 303 rxq->pkts += nb_rx; 304 if (rxq->dirty != old_dirty) { 305 rte_wmb(); 306 idx = AXGBE_GET_DESC_IDX(rxq, rxq->dirty - 1); 307 AXGMAC_DMA_IOWRITE(rxq, DMA_CH_RDTR_LO, 308 low32_value(rxq->ring_phys_addr + 309 (idx * sizeof(union axgbe_rx_desc)))); 310 } 311 312 return nb_rx; 313 } 314 315 316 uint16_t eth_axgbe_recv_scattered_pkts(void *rx_queue, 317 struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 318 { 319 PMD_INIT_FUNC_TRACE(); 320 uint16_t nb_rx = 0; 321 struct axgbe_rx_queue *rxq = rx_queue; 322 volatile union axgbe_rx_desc *desc; 323 324 uint64_t old_dirty = rxq->dirty; 325 struct rte_mbuf *first_seg = NULL; 326 struct rte_mbuf *mbuf, *tmbuf; 327 unsigned int err; 328 uint32_t error_status; 329 uint16_t idx, pidx, data_len = 0, pkt_len = 0; 330 331 idx = AXGBE_GET_DESC_IDX(rxq, rxq->cur); 332 while (nb_rx < nb_pkts) { 333 bool eop = 0; 334 next_desc: 335 if (unlikely(idx == rxq->nb_desc)) 336 idx = 0; 337 338 desc = &rxq->desc[idx]; 339 340 if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, OWN)) 341 break; 342 343 tmbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 344 if (unlikely(!tmbuf)) { 345 PMD_DRV_LOG(ERR, "RX mbuf alloc failed port_id = %u" 346 " queue_id = %u\n", 347 (unsigned int)rxq->port_id, 348 (unsigned int)rxq->queue_id); 349 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 350 break; 351 } 352 353 pidx = idx + 1; 354 if (unlikely(pidx == rxq->nb_desc)) 355 pidx = 0; 356 357 rte_prefetch0(rxq->sw_ring[pidx]); 358 if ((pidx & 0x3) == 0) { 359 rte_prefetch0(&rxq->desc[pidx]); 360 rte_prefetch0(&rxq->sw_ring[pidx]); 361 } 362 363 mbuf = rxq->sw_ring[idx]; 364 /* Check for any errors and free mbuf*/ 365 err = AXGMAC_GET_BITS_LE(desc->write.desc3, 366 RX_NORMAL_DESC3, ES); 367 error_status = 0; 368 if (unlikely(err)) { 369 error_status = desc->write.desc3 & AXGBE_ERR_STATUS; 370 if ((error_status != AXGBE_L3_CSUM_ERR) 371 && (error_status != AXGBE_L4_CSUM_ERR)) { 372 rxq->errors++; 373 rte_pktmbuf_free(mbuf); 374 goto err_set; 375 } 376 } 377 rte_prefetch1(rte_pktmbuf_mtod(mbuf, void *)); 378 379 if (!AXGMAC_GET_BITS_LE(desc->write.desc3, 380 RX_NORMAL_DESC3, LD)) { 381 eop = 0; 382 pkt_len = rxq->buf_size; 383 data_len = pkt_len; 384 } else { 385 eop = 1; 386 pkt_len = AXGMAC_GET_BITS_LE(desc->write.desc3, 387 RX_NORMAL_DESC3, PL); 388 data_len = pkt_len - rxq->crc_len; 389 } 390 391 if (first_seg != NULL) { 392 if (rte_pktmbuf_chain(first_seg, mbuf) != 0) 393 rte_mempool_put(rxq->mb_pool, 394 first_seg); 395 } else { 396 first_seg = mbuf; 397 } 398 399 /* Get the RSS hash */ 400 if (AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, RSV)) 401 mbuf->hash.rss = rte_le_to_cpu_32(desc->write.desc1); 402 403 /* Mbuf populate */ 404 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 405 mbuf->data_len = data_len; 406 407 err_set: 408 rxq->cur++; 409 rxq->sw_ring[idx++] = tmbuf; 410 desc->read.baddr = 411 rte_cpu_to_le_64(rte_mbuf_data_iova_default(tmbuf)); 412 memset((void *)(&desc->read.desc2), 0, 8); 413 AXGMAC_SET_BITS_LE(desc->read.desc3, RX_NORMAL_DESC3, OWN, 1); 414 rxq->dirty++; 415 416 if (!eop) { 417 rte_pktmbuf_free(mbuf); 418 goto next_desc; 419 } 420 421 first_seg->pkt_len = pkt_len; 422 rxq->bytes += pkt_len; 423 mbuf->next = NULL; 424 425 first_seg->port = rxq->port_id; 426 if (rxq->pdata->rx_csum_enable) { 427 mbuf->ol_flags = 0; 428 mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 429 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 430 if (unlikely(error_status == AXGBE_L3_CSUM_ERR)) { 431 mbuf->ol_flags &= ~PKT_RX_IP_CKSUM_GOOD; 432 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 433 mbuf->ol_flags &= ~PKT_RX_L4_CKSUM_GOOD; 434 mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 435 } else if (unlikely(error_status 436 == AXGBE_L4_CSUM_ERR)) { 437 mbuf->ol_flags &= ~PKT_RX_L4_CKSUM_GOOD; 438 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 439 } 440 } 441 442 rx_pkts[nb_rx++] = first_seg; 443 444 /* Setup receipt context for a new packet.*/ 445 first_seg = NULL; 446 } 447 448 /* Save receive context.*/ 449 rxq->pkts += nb_rx; 450 451 if (rxq->dirty != old_dirty) { 452 rte_wmb(); 453 idx = AXGBE_GET_DESC_IDX(rxq, rxq->dirty - 1); 454 AXGMAC_DMA_IOWRITE(rxq, DMA_CH_RDTR_LO, 455 low32_value(rxq->ring_phys_addr + 456 (idx * sizeof(union axgbe_rx_desc)))); 457 } 458 return nb_rx; 459 } 460 461 /* Tx Apis */ 462 static void axgbe_tx_queue_release(struct axgbe_tx_queue *tx_queue) 463 { 464 uint16_t i; 465 struct rte_mbuf **sw_ring; 466 467 if (tx_queue) { 468 sw_ring = tx_queue->sw_ring; 469 if (sw_ring) { 470 for (i = 0; i < tx_queue->nb_desc; i++) { 471 if (sw_ring[i]) 472 rte_pktmbuf_free(sw_ring[i]); 473 } 474 rte_free(sw_ring); 475 } 476 rte_free(tx_queue); 477 } 478 } 479 480 void axgbe_dev_tx_queue_release(void *txq) 481 { 482 axgbe_tx_queue_release(txq); 483 } 484 485 int axgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 486 uint16_t nb_desc, unsigned int socket_id, 487 const struct rte_eth_txconf *tx_conf) 488 { 489 PMD_INIT_FUNC_TRACE(); 490 uint32_t tx_desc; 491 struct axgbe_port *pdata; 492 struct axgbe_tx_queue *txq; 493 unsigned int tsize; 494 const struct rte_memzone *tz; 495 496 tx_desc = nb_desc; 497 pdata = dev->data->dev_private; 498 499 /* 500 * validate tx descriptors count 501 * should be power of 2 and less than h/w supported 502 */ 503 if ((!rte_is_power_of_2(tx_desc)) || 504 tx_desc > pdata->tx_desc_count || 505 tx_desc < AXGBE_MIN_RING_DESC) 506 return -EINVAL; 507 508 /* First allocate the tx queue data structure */ 509 txq = rte_zmalloc("ethdev TX queue", sizeof(struct axgbe_tx_queue), 510 RTE_CACHE_LINE_SIZE); 511 if (!txq) 512 return -ENOMEM; 513 txq->pdata = pdata; 514 515 txq->nb_desc = tx_desc; 516 txq->free_thresh = tx_conf->tx_free_thresh ? 517 tx_conf->tx_free_thresh : AXGBE_TX_FREE_THRESH; 518 if (txq->free_thresh > txq->nb_desc) 519 txq->free_thresh = (txq->nb_desc >> 1); 520 txq->free_batch_cnt = txq->free_thresh; 521 522 /* In vector_tx path threshold should be multiple of queue_size*/ 523 if (txq->nb_desc % txq->free_thresh != 0) 524 txq->vector_disable = 1; 525 526 if (tx_conf->offloads != 0) 527 txq->vector_disable = 1; 528 529 /* Allocate TX ring hardware descriptors */ 530 tsize = txq->nb_desc * sizeof(struct axgbe_tx_desc); 531 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 532 tsize, AXGBE_DESC_ALIGN, socket_id); 533 if (!tz) { 534 axgbe_tx_queue_release(txq); 535 return -ENOMEM; 536 } 537 memset(tz->addr, 0, tsize); 538 txq->ring_phys_addr = (uint64_t)tz->iova; 539 txq->desc = tz->addr; 540 txq->queue_id = queue_idx; 541 txq->port_id = dev->data->port_id; 542 txq->dma_regs = (void *)((uint8_t *)pdata->xgmac_regs + DMA_CH_BASE + 543 (DMA_CH_INC * txq->queue_id)); 544 txq->dma_tail_reg = (volatile uint32_t *)((uint8_t *)txq->dma_regs + 545 DMA_CH_TDTR_LO); 546 txq->cur = 0; 547 txq->dirty = 0; 548 txq->nb_desc_free = txq->nb_desc; 549 /* Allocate software ring */ 550 tsize = txq->nb_desc * sizeof(struct rte_mbuf *); 551 txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize, 552 RTE_CACHE_LINE_SIZE); 553 if (!txq->sw_ring) { 554 axgbe_tx_queue_release(txq); 555 return -ENOMEM; 556 } 557 dev->data->tx_queues[queue_idx] = txq; 558 if (!pdata->tx_queues) 559 pdata->tx_queues = dev->data->tx_queues; 560 561 if (txq->vector_disable || 562 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 563 dev->tx_pkt_burst = &axgbe_xmit_pkts; 564 else 565 #ifdef RTE_ARCH_X86 566 dev->tx_pkt_burst = &axgbe_xmit_pkts_vec; 567 #else 568 dev->tx_pkt_burst = &axgbe_xmit_pkts; 569 #endif 570 571 return 0; 572 } 573 574 static void axgbe_txq_prepare_tx_stop(struct axgbe_port *pdata, 575 unsigned int queue) 576 { 577 unsigned int tx_status; 578 unsigned long tx_timeout; 579 580 /* The Tx engine cannot be stopped if it is actively processing 581 * packets. Wait for the Tx queue to empty the Tx fifo. Don't 582 * wait forever though... 583 */ 584 tx_timeout = rte_get_timer_cycles() + (AXGBE_DMA_STOP_TIMEOUT * 585 rte_get_timer_hz()); 586 while (time_before(rte_get_timer_cycles(), tx_timeout)) { 587 tx_status = AXGMAC_MTL_IOREAD(pdata, queue, MTL_Q_TQDR); 588 if ((AXGMAC_GET_BITS(tx_status, MTL_Q_TQDR, TRCSTS) != 1) && 589 (AXGMAC_GET_BITS(tx_status, MTL_Q_TQDR, TXQSTS) == 0)) 590 break; 591 592 rte_delay_us(900); 593 } 594 595 if (!time_before(rte_get_timer_cycles(), tx_timeout)) 596 PMD_DRV_LOG(ERR, 597 "timed out waiting for Tx queue %u to empty\n", 598 queue); 599 } 600 601 static void axgbe_prepare_tx_stop(struct axgbe_port *pdata, 602 unsigned int queue) 603 { 604 unsigned int tx_dsr, tx_pos, tx_qidx; 605 unsigned int tx_status; 606 unsigned long tx_timeout; 607 608 if (AXGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER) > 0x20) 609 return axgbe_txq_prepare_tx_stop(pdata, queue); 610 611 /* Calculate the status register to read and the position within */ 612 if (queue < DMA_DSRX_FIRST_QUEUE) { 613 tx_dsr = DMA_DSR0; 614 tx_pos = (queue * DMA_DSR_Q_WIDTH) + DMA_DSR0_TPS_START; 615 } else { 616 tx_qidx = queue - DMA_DSRX_FIRST_QUEUE; 617 618 tx_dsr = DMA_DSR1 + ((tx_qidx / DMA_DSRX_QPR) * DMA_DSRX_INC); 619 tx_pos = ((tx_qidx % DMA_DSRX_QPR) * DMA_DSR_Q_WIDTH) + 620 DMA_DSRX_TPS_START; 621 } 622 623 /* The Tx engine cannot be stopped if it is actively processing 624 * descriptors. Wait for the Tx engine to enter the stopped or 625 * suspended state. Don't wait forever though... 626 */ 627 tx_timeout = rte_get_timer_cycles() + (AXGBE_DMA_STOP_TIMEOUT * 628 rte_get_timer_hz()); 629 while (time_before(rte_get_timer_cycles(), tx_timeout)) { 630 tx_status = AXGMAC_IOREAD(pdata, tx_dsr); 631 tx_status = GET_BITS(tx_status, tx_pos, DMA_DSR_TPS_WIDTH); 632 if ((tx_status == DMA_TPS_STOPPED) || 633 (tx_status == DMA_TPS_SUSPENDED)) 634 break; 635 636 rte_delay_us(900); 637 } 638 639 if (!time_before(rte_get_timer_cycles(), tx_timeout)) 640 PMD_DRV_LOG(ERR, 641 "timed out waiting for Tx DMA channel %u to stop\n", 642 queue); 643 } 644 645 void axgbe_dev_disable_tx(struct rte_eth_dev *dev) 646 { 647 struct axgbe_tx_queue *txq; 648 struct axgbe_port *pdata = dev->data->dev_private; 649 unsigned int i; 650 651 /* Prepare for stopping DMA channel */ 652 for (i = 0; i < pdata->tx_q_count; i++) { 653 txq = dev->data->tx_queues[i]; 654 axgbe_prepare_tx_stop(pdata, i); 655 } 656 /* Disable MAC Tx */ 657 AXGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 0); 658 /* Disable each Tx queue*/ 659 for (i = 0; i < pdata->tx_q_count; i++) 660 AXGMAC_MTL_IOWRITE_BITS(pdata, i, MTL_Q_TQOMR, TXQEN, 661 0); 662 /* Disable each Tx DMA channel */ 663 for (i = 0; i < dev->data->nb_tx_queues; i++) { 664 txq = dev->data->tx_queues[i]; 665 AXGMAC_DMA_IOWRITE_BITS(txq, DMA_CH_TCR, ST, 0); 666 } 667 } 668 669 void axgbe_dev_enable_tx(struct rte_eth_dev *dev) 670 { 671 struct axgbe_tx_queue *txq; 672 struct axgbe_port *pdata = dev->data->dev_private; 673 unsigned int i; 674 675 for (i = 0; i < dev->data->nb_tx_queues; i++) { 676 txq = dev->data->tx_queues[i]; 677 /* Enable Tx DMA channel */ 678 AXGMAC_DMA_IOWRITE_BITS(txq, DMA_CH_TCR, ST, 1); 679 } 680 /* Enable Tx queue*/ 681 for (i = 0; i < pdata->tx_q_count; i++) 682 AXGMAC_MTL_IOWRITE_BITS(pdata, i, MTL_Q_TQOMR, TXQEN, 683 MTL_Q_ENABLED); 684 /* Enable MAC Tx */ 685 AXGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 1); 686 } 687 688 /* Free Tx conformed mbufs */ 689 static void axgbe_xmit_cleanup(struct axgbe_tx_queue *txq) 690 { 691 volatile struct axgbe_tx_desc *desc; 692 uint16_t idx; 693 694 idx = AXGBE_GET_DESC_IDX(txq, txq->dirty); 695 while (txq->cur != txq->dirty) { 696 if (unlikely(idx == txq->nb_desc)) 697 idx = 0; 698 desc = &txq->desc[idx]; 699 /* Check for ownership */ 700 if (AXGMAC_GET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, OWN)) 701 return; 702 memset((void *)&desc->desc2, 0, 8); 703 /* Free mbuf */ 704 rte_pktmbuf_free(txq->sw_ring[idx]); 705 txq->sw_ring[idx++] = NULL; 706 txq->dirty++; 707 } 708 } 709 710 /* Tx Descriptor formation 711 * Considering each mbuf requires one desc 712 * mbuf is linear 713 */ 714 static int axgbe_xmit_hw(struct axgbe_tx_queue *txq, 715 struct rte_mbuf *mbuf) 716 { 717 volatile struct axgbe_tx_desc *desc; 718 uint16_t idx; 719 uint64_t mask; 720 721 idx = AXGBE_GET_DESC_IDX(txq, txq->cur); 722 desc = &txq->desc[idx]; 723 724 /* Update buffer address and length */ 725 desc->baddr = rte_mbuf_data_iova(mbuf); 726 AXGMAC_SET_BITS_LE(desc->desc2, TX_NORMAL_DESC2, HL_B1L, 727 mbuf->pkt_len); 728 /* Total msg length to transmit */ 729 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, FL, 730 mbuf->pkt_len); 731 /* Timestamp enablement check */ 732 if (mbuf->ol_flags & PKT_TX_IEEE1588_TMST) 733 AXGMAC_SET_BITS_LE(desc->desc2, TX_NORMAL_DESC2, TTSE, 1); 734 rte_wmb(); 735 /* Mark it as First and Last Descriptor */ 736 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, FD, 1); 737 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, LD, 1); 738 /* Mark it as a NORMAL descriptor */ 739 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, CTXT, 0); 740 /* configure h/w Offload */ 741 mask = mbuf->ol_flags & PKT_TX_L4_MASK; 742 if ((mask == PKT_TX_TCP_CKSUM) || (mask == PKT_TX_UDP_CKSUM)) 743 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, CIC, 0x3); 744 else if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 745 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, CIC, 0x1); 746 rte_wmb(); 747 748 /* Set OWN bit */ 749 AXGMAC_SET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, OWN, 1); 750 rte_wmb(); 751 752 /* Save mbuf */ 753 txq->sw_ring[idx] = mbuf; 754 /* Update current index*/ 755 txq->cur++; 756 /* Update stats */ 757 txq->bytes += mbuf->pkt_len; 758 759 return 0; 760 } 761 762 /* Eal supported tx wrapper*/ 763 uint16_t 764 axgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 765 uint16_t nb_pkts) 766 { 767 PMD_INIT_FUNC_TRACE(); 768 769 if (unlikely(nb_pkts == 0)) 770 return nb_pkts; 771 772 struct axgbe_tx_queue *txq; 773 uint16_t nb_desc_free; 774 uint16_t nb_pkt_sent = 0; 775 uint16_t idx; 776 uint32_t tail_addr; 777 struct rte_mbuf *mbuf; 778 779 txq = (struct axgbe_tx_queue *)tx_queue; 780 nb_desc_free = txq->nb_desc - (txq->cur - txq->dirty); 781 782 if (unlikely(nb_desc_free <= txq->free_thresh)) { 783 axgbe_xmit_cleanup(txq); 784 nb_desc_free = txq->nb_desc - (txq->cur - txq->dirty); 785 if (unlikely(nb_desc_free == 0)) 786 return 0; 787 } 788 nb_pkts = RTE_MIN(nb_desc_free, nb_pkts); 789 while (nb_pkts--) { 790 mbuf = *tx_pkts++; 791 if (axgbe_xmit_hw(txq, mbuf)) 792 goto out; 793 nb_pkt_sent++; 794 } 795 out: 796 /* Sync read and write */ 797 rte_mb(); 798 idx = AXGBE_GET_DESC_IDX(txq, txq->cur); 799 tail_addr = low32_value(txq->ring_phys_addr + 800 idx * sizeof(struct axgbe_tx_desc)); 801 /* Update tail reg with next immediate address to kick Tx DMA channel*/ 802 AXGMAC_DMA_IOWRITE(txq, DMA_CH_TDTR_LO, tail_addr); 803 txq->pkts += nb_pkt_sent; 804 return nb_pkt_sent; 805 } 806 807 void axgbe_dev_clear_queues(struct rte_eth_dev *dev) 808 { 809 PMD_INIT_FUNC_TRACE(); 810 uint8_t i; 811 struct axgbe_rx_queue *rxq; 812 struct axgbe_tx_queue *txq; 813 814 for (i = 0; i < dev->data->nb_rx_queues; i++) { 815 rxq = dev->data->rx_queues[i]; 816 817 if (rxq) { 818 axgbe_rx_queue_release(rxq); 819 dev->data->rx_queues[i] = NULL; 820 } 821 } 822 823 for (i = 0; i < dev->data->nb_tx_queues; i++) { 824 txq = dev->data->tx_queues[i]; 825 826 if (txq) { 827 axgbe_tx_queue_release(txq); 828 dev->data->tx_queues[i] = NULL; 829 } 830 } 831 } 832 833 int 834 axgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 835 { 836 struct axgbe_rx_queue *rxq = rx_queue; 837 volatile union axgbe_rx_desc *desc; 838 uint16_t idx; 839 840 841 if (unlikely(offset >= rxq->nb_desc)) 842 return -EINVAL; 843 844 if (offset >= rxq->nb_desc - rxq->dirty) 845 return RTE_ETH_RX_DESC_UNAVAIL; 846 847 idx = AXGBE_GET_DESC_IDX(rxq, rxq->cur); 848 desc = &rxq->desc[idx + offset]; 849 850 if (!AXGMAC_GET_BITS_LE(desc->write.desc3, RX_NORMAL_DESC3, OWN)) 851 return RTE_ETH_RX_DESC_DONE; 852 853 return RTE_ETH_RX_DESC_AVAIL; 854 } 855 856 int 857 axgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 858 { 859 struct axgbe_tx_queue *txq = tx_queue; 860 volatile struct axgbe_tx_desc *desc; 861 uint16_t idx; 862 863 864 if (unlikely(offset >= txq->nb_desc)) 865 return -EINVAL; 866 867 if (offset >= txq->nb_desc - txq->dirty) 868 return RTE_ETH_TX_DESC_UNAVAIL; 869 870 idx = AXGBE_GET_DESC_IDX(txq, txq->dirty + txq->free_batch_cnt - 1); 871 desc = &txq->desc[idx + offset]; 872 873 if (!AXGMAC_GET_BITS_LE(desc->desc3, TX_NORMAL_DESC3, OWN)) 874 return RTE_ETH_TX_DESC_DONE; 875 876 return RTE_ETH_TX_DESC_FULL; 877 } 878