1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <stdint.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 #include <rte_log.h> 19 #include <rte_debug.h> 20 #include <rte_interrupts.h> 21 #include <rte_pci.h> 22 #include <rte_memory.h> 23 #include <rte_memzone.h> 24 #include <rte_launch.h> 25 #include <rte_eal.h> 26 #include <rte_per_lcore.h> 27 #include <rte_lcore.h> 28 #include <rte_atomic.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_mempool.h> 31 #include <rte_malloc.h> 32 #include <rte_mbuf.h> 33 #include <rte_ether.h> 34 #include <ethdev_driver.h> 35 #include <rte_prefetch.h> 36 #include <rte_udp.h> 37 #include <rte_tcp.h> 38 #include <rte_sctp.h> 39 #include <rte_string_fns.h> 40 #include <rte_errno.h> 41 #include <rte_ip.h> 42 #include <rte_net.h> 43 44 #include "ionic_logs.h" 45 #include "ionic_mac_api.h" 46 #include "ionic_ethdev.h" 47 #include "ionic_lif.h" 48 #include "ionic_rxtx.h" 49 50 #define IONIC_RX_RING_DOORBELL_STRIDE (32 - 1) 51 52 /********************************************************************* 53 * 54 * TX functions 55 * 56 **********************************************************************/ 57 58 void 59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 60 struct rte_eth_txq_info *qinfo) 61 { 62 struct ionic_qcq *txq = dev->data->tx_queues[queue_id]; 63 struct ionic_queue *q = &txq->q; 64 65 qinfo->nb_desc = q->num_descs; 66 qinfo->conf.offloads = txq->offloads; 67 qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED; 68 } 69 70 static inline void __rte_cold 71 ionic_tx_flush(struct ionic_cq *cq) 72 { 73 struct ionic_queue *q = cq->bound_q; 74 struct ionic_desc_info *q_desc_info; 75 struct rte_mbuf *txm, *next; 76 struct ionic_txq_comp *cq_desc_base = cq->base; 77 struct ionic_txq_comp *cq_desc; 78 u_int32_t comp_index = (u_int32_t)-1; 79 80 cq_desc = &cq_desc_base[cq->tail_idx]; 81 while (color_match(cq_desc->color, cq->done_color)) { 82 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 83 84 /* Prefetch the next 4 descriptors (not really useful here) */ 85 if ((cq->tail_idx & 0x3) == 0) 86 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 87 88 if (cq->tail_idx == 0) 89 cq->done_color = !cq->done_color; 90 91 comp_index = cq_desc->comp_index; 92 93 cq_desc = &cq_desc_base[cq->tail_idx]; 94 } 95 96 if (comp_index != (u_int32_t)-1) { 97 while (q->tail_idx != comp_index) { 98 q_desc_info = &q->info[q->tail_idx]; 99 100 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 101 102 /* Prefetch the next 4 descriptors */ 103 if ((q->tail_idx & 0x3) == 0) 104 /* q desc info */ 105 rte_prefetch0(&q->info[q->tail_idx]); 106 107 /* 108 * Note: you can just use rte_pktmbuf_free, 109 * but this loop is faster 110 */ 111 txm = q_desc_info->cb_arg; 112 while (txm != NULL) { 113 next = txm->next; 114 rte_pktmbuf_free_seg(txm); 115 txm = next; 116 } 117 } 118 } 119 } 120 121 void __rte_cold 122 ionic_dev_tx_queue_release(void *tx_queue) 123 { 124 struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; 125 126 IONIC_PRINT_CALL(); 127 128 ionic_lif_txq_deinit(txq); 129 130 ionic_qcq_free(txq); 131 } 132 133 int __rte_cold 134 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 135 { 136 struct ionic_qcq *txq; 137 138 IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); 139 140 txq = eth_dev->data->tx_queues[tx_queue_id]; 141 142 eth_dev->data->tx_queue_state[tx_queue_id] = 143 RTE_ETH_QUEUE_STATE_STOPPED; 144 145 /* 146 * Note: we should better post NOP Tx desc and wait for its completion 147 * before disabling Tx queue 148 */ 149 150 ionic_qcq_disable(txq); 151 152 ionic_tx_flush(&txq->cq); 153 154 return 0; 155 } 156 157 int __rte_cold 158 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, 159 uint16_t nb_desc, uint32_t socket_id, 160 const struct rte_eth_txconf *tx_conf) 161 { 162 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 163 struct ionic_qcq *txq; 164 uint64_t offloads; 165 int err; 166 167 if (tx_queue_id >= lif->ntxqcqs) { 168 IONIC_PRINT(DEBUG, "Queue index %u not available " 169 "(max %u queues)", 170 tx_queue_id, lif->ntxqcqs); 171 return -EINVAL; 172 } 173 174 offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads; 175 IONIC_PRINT(DEBUG, 176 "Configuring skt %u TX queue %u with %u buffers, offloads %jx", 177 socket_id, tx_queue_id, nb_desc, offloads); 178 179 /* Validate number of receive descriptors */ 180 if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC) 181 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 182 183 /* Free memory prior to re-allocation if needed... */ 184 if (eth_dev->data->tx_queues[tx_queue_id] != NULL) { 185 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id]; 186 ionic_dev_tx_queue_release(tx_queue); 187 eth_dev->data->tx_queues[tx_queue_id] = NULL; 188 } 189 190 eth_dev->data->tx_queue_state[tx_queue_id] = 191 RTE_ETH_QUEUE_STATE_STOPPED; 192 193 err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq); 194 if (err) { 195 IONIC_PRINT(DEBUG, "Queue allocation failure"); 196 return -EINVAL; 197 } 198 199 /* Do not start queue with rte_eth_dev_start() */ 200 if (tx_conf->tx_deferred_start) 201 txq->flags |= IONIC_QCQ_F_DEFERRED; 202 203 txq->offloads = offloads; 204 205 eth_dev->data->tx_queues[tx_queue_id] = txq; 206 207 return 0; 208 } 209 210 /* 211 * Start Transmit Units for specified queue. 212 */ 213 int __rte_cold 214 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 215 { 216 uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; 217 struct ionic_qcq *txq; 218 int err; 219 220 if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 221 IONIC_PRINT(DEBUG, "TX queue %u already started", 222 tx_queue_id); 223 return 0; 224 } 225 226 txq = eth_dev->data->tx_queues[tx_queue_id]; 227 228 IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", 229 tx_queue_id, txq->q.num_descs); 230 231 if (!(txq->flags & IONIC_QCQ_F_INITED)) { 232 err = ionic_lif_txq_init(txq); 233 if (err) 234 return err; 235 } else { 236 ionic_qcq_enable(txq); 237 } 238 239 tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 240 241 return 0; 242 } 243 244 static void 245 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm) 246 { 247 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 248 char *l3_hdr = ((char *)eth_hdr) + txm->l2_len; 249 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 250 (l3_hdr + txm->l3_len); 251 252 if (txm->ol_flags & PKT_TX_IP_CKSUM) { 253 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 254 ipv4_hdr->hdr_checksum = 0; 255 tcp_hdr->cksum = 0; 256 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 257 } else { 258 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 259 tcp_hdr->cksum = 0; 260 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 261 } 262 } 263 264 static void 265 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm) 266 { 267 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 268 char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len + 269 txm->outer_l3_len + txm->l2_len; 270 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 271 (l3_hdr + txm->l3_len); 272 273 if (txm->ol_flags & PKT_TX_IPV4) { 274 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 275 ipv4_hdr->hdr_checksum = 0; 276 tcp_hdr->cksum = 0; 277 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 278 } else { 279 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 280 tcp_hdr->cksum = 0; 281 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 282 } 283 } 284 285 static void 286 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 287 struct rte_mbuf *txm, 288 rte_iova_t addr, uint8_t nsge, uint16_t len, 289 uint32_t hdrlen, uint32_t mss, 290 bool encap, 291 uint16_t vlan_tci, bool has_vlan, 292 bool start, bool done) 293 { 294 uint8_t flags = 0; 295 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 296 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 297 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 298 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 299 300 desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, 301 flags, nsge, addr); 302 desc->len = len; 303 desc->vlan_tci = vlan_tci; 304 desc->hdr_len = hdrlen; 305 desc->mss = mss; 306 307 ionic_q_post(q, done, NULL, done ? txm : NULL); 308 } 309 310 static struct ionic_txq_desc * 311 ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) 312 { 313 struct ionic_txq_desc *desc_base = q->base; 314 struct ionic_txq_sg_desc *sg_desc_base = q->sg_base; 315 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 316 struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx]; 317 318 *elem = sg_desc->elems; 319 return desc; 320 } 321 322 static int 323 ionic_tx_tso(struct ionic_queue *q, struct rte_mbuf *txm, 324 uint64_t offloads __rte_unused, bool not_xmit_more) 325 { 326 struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); 327 struct ionic_txq_desc *desc; 328 struct ionic_txq_sg_elem *elem; 329 struct rte_mbuf *txm_seg; 330 uint64_t desc_addr = 0; 331 uint16_t desc_len = 0; 332 uint8_t desc_nsge; 333 uint32_t hdrlen; 334 uint32_t mss = txm->tso_segsz; 335 uint32_t frag_left = 0; 336 uint32_t left; 337 uint32_t seglen; 338 uint32_t len; 339 uint32_t offset = 0; 340 bool start, done; 341 bool encap; 342 bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT); 343 uint16_t vlan_tci = txm->vlan_tci; 344 uint64_t ol_flags = txm->ol_flags; 345 346 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 347 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 348 ((ol_flags & PKT_TX_OUTER_IPV4) || 349 (ol_flags & PKT_TX_OUTER_IPV6)); 350 351 /* Preload inner-most TCP csum field with IP pseudo hdr 352 * calculated with IP length set to zero. HW will later 353 * add in length to each TCP segment resulting from the TSO. 354 */ 355 356 if (encap) { 357 ionic_tx_tcp_inner_pseudo_csum(txm); 358 hdrlen = txm->outer_l2_len + txm->outer_l3_len + 359 txm->l2_len + txm->l3_len + txm->l4_len; 360 } else { 361 ionic_tx_tcp_pseudo_csum(txm); 362 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len; 363 } 364 365 seglen = hdrlen + mss; 366 left = txm->data_len; 367 368 desc = ionic_tx_tso_next(q, &elem); 369 start = true; 370 371 /* Chop data up into desc segments */ 372 373 while (left > 0) { 374 len = RTE_MIN(seglen, left); 375 frag_left = seglen - len; 376 desc_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm)); 377 desc_len = len; 378 desc_nsge = 0; 379 left -= len; 380 offset += len; 381 if (txm->nb_segs > 1 && frag_left > 0) 382 continue; 383 done = (txm->nb_segs == 1 && left == 0); 384 ionic_tx_tso_post(q, desc, txm, 385 desc_addr, desc_nsge, desc_len, 386 hdrlen, mss, 387 encap, 388 vlan_tci, has_vlan, 389 start, done && not_xmit_more); 390 desc = ionic_tx_tso_next(q, &elem); 391 start = false; 392 seglen = mss; 393 } 394 395 /* Chop frags into desc segments */ 396 397 txm_seg = txm->next; 398 while (txm_seg != NULL) { 399 offset = 0; 400 left = txm_seg->data_len; 401 stats->frags++; 402 403 while (left > 0) { 404 rte_iova_t data_iova; 405 data_iova = rte_mbuf_data_iova(txm_seg); 406 elem->addr = rte_cpu_to_le_64(data_iova) + offset; 407 if (frag_left > 0) { 408 len = RTE_MIN(frag_left, left); 409 frag_left -= len; 410 elem->len = len; 411 elem++; 412 desc_nsge++; 413 } else { 414 len = RTE_MIN(mss, left); 415 frag_left = mss - len; 416 data_iova = rte_mbuf_data_iova(txm_seg); 417 desc_addr = rte_cpu_to_le_64(data_iova); 418 desc_len = len; 419 desc_nsge = 0; 420 } 421 left -= len; 422 offset += len; 423 if (txm_seg->next != NULL && frag_left > 0) 424 continue; 425 done = (txm_seg->next == NULL && left == 0); 426 ionic_tx_tso_post(q, desc, txm_seg, 427 desc_addr, desc_nsge, desc_len, 428 hdrlen, mss, 429 encap, 430 vlan_tci, has_vlan, 431 start, done && not_xmit_more); 432 desc = ionic_tx_tso_next(q, &elem); 433 start = false; 434 } 435 436 txm_seg = txm_seg->next; 437 } 438 439 stats->tso++; 440 441 return 0; 442 } 443 444 static int 445 ionic_tx(struct ionic_queue *q, struct rte_mbuf *txm, 446 uint64_t offloads, bool not_xmit_more) 447 { 448 struct ionic_txq_desc *desc_base = q->base; 449 struct ionic_txq_sg_desc *sg_desc_base = q->sg_base; 450 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 451 struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx]; 452 struct ionic_txq_sg_elem *elem = sg_desc->elems; 453 struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); 454 struct rte_mbuf *txm_seg; 455 bool encap; 456 bool has_vlan; 457 uint64_t ol_flags = txm->ol_flags; 458 uint64_t addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm)); 459 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 460 uint8_t flags = 0; 461 462 if ((ol_flags & PKT_TX_IP_CKSUM) && 463 (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 464 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 465 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 466 } 467 468 if (((ol_flags & PKT_TX_TCP_CKSUM) && 469 (offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) || 470 ((ol_flags & PKT_TX_UDP_CKSUM) && 471 (offloads & DEV_TX_OFFLOAD_UDP_CKSUM))) { 472 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 473 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 474 } 475 476 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 477 stats->no_csum++; 478 479 has_vlan = (ol_flags & PKT_TX_VLAN_PKT); 480 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 481 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 482 ((ol_flags & PKT_TX_OUTER_IPV4) || 483 (ol_flags & PKT_TX_OUTER_IPV6)); 484 485 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 486 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 487 488 desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 489 desc->len = txm->data_len; 490 desc->vlan_tci = txm->vlan_tci; 491 492 txm_seg = txm->next; 493 while (txm_seg != NULL) { 494 elem->len = txm_seg->data_len; 495 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); 496 stats->frags++; 497 elem++; 498 txm_seg = txm_seg->next; 499 } 500 501 ionic_q_post(q, not_xmit_more, NULL, txm); 502 503 return 0; 504 } 505 506 uint16_t 507 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 508 uint16_t nb_pkts) 509 { 510 struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; 511 struct ionic_queue *q = &txq->q; 512 struct ionic_cq *cq = &txq->cq; 513 struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); 514 uint32_t next_q_head_idx; 515 uint32_t bytes_tx = 0; 516 uint16_t nb_tx = 0; 517 int err; 518 bool last; 519 520 /* Cleaning old buffers */ 521 ionic_tx_flush(cq); 522 523 if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { 524 stats->stop += nb_pkts; 525 return 0; 526 } 527 528 while (nb_tx < nb_pkts) { 529 last = (nb_tx == (nb_pkts - 1)); 530 531 next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1); 532 if ((next_q_head_idx & 0x3) == 0) { 533 struct ionic_txq_desc *desc_base = q->base; 534 rte_prefetch0(&desc_base[next_q_head_idx]); 535 rte_prefetch0(&q->info[next_q_head_idx]); 536 } 537 538 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG) 539 err = ionic_tx_tso(q, tx_pkts[nb_tx], txq->offloads, 540 last); 541 else 542 err = ionic_tx(q, tx_pkts[nb_tx], txq->offloads, last); 543 if (err) { 544 stats->drop += nb_pkts - nb_tx; 545 if (nb_tx > 0) 546 ionic_q_flush(q); 547 break; 548 } 549 550 bytes_tx += tx_pkts[nb_tx]->pkt_len; 551 nb_tx++; 552 } 553 554 stats->packets += nb_tx; 555 stats->bytes += bytes_tx; 556 557 return nb_tx; 558 } 559 560 /********************************************************************* 561 * 562 * TX prep functions 563 * 564 **********************************************************************/ 565 566 #define IONIC_TX_OFFLOAD_MASK ( \ 567 PKT_TX_IPV4 | \ 568 PKT_TX_IPV6 | \ 569 PKT_TX_VLAN | \ 570 PKT_TX_IP_CKSUM | \ 571 PKT_TX_TCP_SEG | \ 572 PKT_TX_L4_MASK) 573 574 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \ 575 (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) 576 577 uint16_t 578 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 579 uint16_t nb_pkts) 580 { 581 struct rte_mbuf *txm; 582 uint64_t offloads; 583 int i = 0; 584 585 for (i = 0; i < nb_pkts; i++) { 586 txm = tx_pkts[i]; 587 588 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS) { 589 rte_errno = -EINVAL; 590 break; 591 } 592 593 offloads = txm->ol_flags; 594 595 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) { 596 rte_errno = -ENOTSUP; 597 break; 598 } 599 } 600 601 return i; 602 } 603 604 /********************************************************************* 605 * 606 * RX functions 607 * 608 **********************************************************************/ 609 610 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 611 struct rte_mbuf *mbuf); 612 613 void 614 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 615 struct rte_eth_rxq_info *qinfo) 616 { 617 struct ionic_qcq *rxq = dev->data->rx_queues[queue_id]; 618 struct ionic_queue *q = &rxq->q; 619 620 qinfo->mp = rxq->mb_pool; 621 qinfo->scattered_rx = dev->data->scattered_rx; 622 qinfo->nb_desc = q->num_descs; 623 qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED; 624 qinfo->conf.offloads = rxq->offloads; 625 } 626 627 static void __rte_cold 628 ionic_rx_empty(struct ionic_queue *q) 629 { 630 struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); 631 struct ionic_desc_info *cur; 632 struct rte_mbuf *mbuf; 633 634 while (q->tail_idx != q->head_idx) { 635 cur = &q->info[q->tail_idx]; 636 mbuf = cur->cb_arg; 637 rte_mempool_put(rxq->mb_pool, mbuf); 638 639 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 640 } 641 } 642 643 void __rte_cold 644 ionic_dev_rx_queue_release(void *rx_queue) 645 { 646 struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; 647 648 IONIC_PRINT_CALL(); 649 650 ionic_rx_empty(&rxq->q); 651 652 ionic_lif_rxq_deinit(rxq); 653 654 ionic_qcq_free(rxq); 655 } 656 657 int __rte_cold 658 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, 659 uint16_t rx_queue_id, 660 uint16_t nb_desc, 661 uint32_t socket_id, 662 const struct rte_eth_rxconf *rx_conf, 663 struct rte_mempool *mp) 664 { 665 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 666 struct ionic_qcq *rxq; 667 uint64_t offloads; 668 int err; 669 670 if (rx_queue_id >= lif->nrxqcqs) { 671 IONIC_PRINT(ERR, 672 "Queue index %u not available (max %u queues)", 673 rx_queue_id, lif->nrxqcqs); 674 return -EINVAL; 675 } 676 677 offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads; 678 IONIC_PRINT(DEBUG, 679 "Configuring skt %u RX queue %u with %u buffers, offloads %jx", 680 socket_id, rx_queue_id, nb_desc, offloads); 681 682 if (!rx_conf->rx_drop_en) 683 IONIC_PRINT(WARNING, "No-drop mode is not supported"); 684 685 /* Validate number of receive descriptors */ 686 if (!rte_is_power_of_2(nb_desc) || 687 nb_desc < IONIC_MIN_RING_DESC || 688 nb_desc > IONIC_MAX_RING_DESC) { 689 IONIC_PRINT(ERR, 690 "Bad descriptor count (%u) for queue %u (min: %u)", 691 nb_desc, rx_queue_id, IONIC_MIN_RING_DESC); 692 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 693 } 694 695 /* Free memory prior to re-allocation if needed... */ 696 if (eth_dev->data->rx_queues[rx_queue_id] != NULL) { 697 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id]; 698 ionic_dev_rx_queue_release(rx_queue); 699 eth_dev->data->rx_queues[rx_queue_id] = NULL; 700 } 701 702 eth_dev->data->rx_queue_state[rx_queue_id] = 703 RTE_ETH_QUEUE_STATE_STOPPED; 704 705 err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq); 706 if (err) { 707 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); 708 return -EINVAL; 709 } 710 711 rxq->mb_pool = mp; 712 713 /* 714 * Note: the interface does not currently support 715 * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN 716 * when the adapter will be able to keep the CRC and subtract 717 * it to the length for all received packets: 718 * if (eth_dev->data->dev_conf.rxmode.offloads & 719 * DEV_RX_OFFLOAD_KEEP_CRC) 720 * rxq->crc_len = ETHER_CRC_LEN; 721 */ 722 723 /* Do not start queue with rte_eth_dev_start() */ 724 if (rx_conf->rx_deferred_start) 725 rxq->flags |= IONIC_QCQ_F_DEFERRED; 726 727 rxq->offloads = offloads; 728 729 eth_dev->data->rx_queues[rx_queue_id] = rxq; 730 731 return 0; 732 } 733 734 static void 735 ionic_rx_clean(struct ionic_queue *q, 736 uint32_t q_desc_index, uint32_t cq_desc_index, 737 void *cb_arg, void *service_cb_arg) 738 { 739 struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base; 740 struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 741 struct rte_mbuf *rxm = cb_arg; 742 struct rte_mbuf *rxm_seg; 743 struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); 744 uint32_t max_frame_size = 745 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 746 uint64_t pkt_flags = 0; 747 uint32_t pkt_type; 748 struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q); 749 struct ionic_rx_service *recv_args = (struct ionic_rx_service *) 750 service_cb_arg; 751 uint32_t buf_size = (uint16_t) 752 (rte_pktmbuf_data_room_size(rxq->mb_pool) - 753 RTE_PKTMBUF_HEADROOM); 754 uint32_t left; 755 756 if (!recv_args) { 757 stats->no_cb_arg++; 758 /* Flush */ 759 rte_pktmbuf_free(rxm); 760 /* 761 * Note: rte_mempool_put is faster with no segs 762 * rte_mempool_put(rxq->mb_pool, rxm); 763 */ 764 return; 765 } 766 767 if (cq_desc->status) { 768 stats->bad_cq_status++; 769 ionic_rx_recycle(q, q_desc_index, rxm); 770 return; 771 } 772 773 if (recv_args->nb_rx >= recv_args->nb_pkts) { 774 stats->no_room++; 775 ionic_rx_recycle(q, q_desc_index, rxm); 776 return; 777 } 778 779 if (cq_desc->len > max_frame_size || 780 cq_desc->len == 0) { 781 stats->bad_len++; 782 ionic_rx_recycle(q, q_desc_index, rxm); 783 return; 784 } 785 786 rxm->data_off = RTE_PKTMBUF_HEADROOM; 787 rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); 788 rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ 789 rxm->pkt_len = cq_desc->len; 790 rxm->port = rxq->lif->port_id; 791 792 left = cq_desc->len; 793 794 rxm->data_len = RTE_MIN(buf_size, left); 795 left -= rxm->data_len; 796 797 rxm_seg = rxm->next; 798 while (rxm_seg && left) { 799 rxm_seg->data_len = RTE_MIN(buf_size, left); 800 left -= rxm_seg->data_len; 801 802 rxm_seg = rxm_seg->next; 803 rxm->nb_segs++; 804 } 805 806 /* RSS */ 807 pkt_flags |= PKT_RX_RSS_HASH; 808 rxm->hash.rss = cq_desc->rss_hash; 809 810 /* Vlan Strip */ 811 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 812 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 813 rxm->vlan_tci = cq_desc->vlan_tci; 814 } 815 816 /* Checksum */ 817 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 818 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK) 819 pkt_flags |= PKT_RX_IP_CKSUM_GOOD; 820 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD) 821 pkt_flags |= PKT_RX_IP_CKSUM_BAD; 822 823 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) || 824 (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK)) 825 pkt_flags |= PKT_RX_L4_CKSUM_GOOD; 826 else if ((cq_desc->csum_flags & 827 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 828 (cq_desc->csum_flags & 829 IONIC_RXQ_COMP_CSUM_F_UDP_BAD)) 830 pkt_flags |= PKT_RX_L4_CKSUM_BAD; 831 } 832 833 rxm->ol_flags = pkt_flags; 834 835 /* Packet Type */ 836 switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 837 case IONIC_PKT_TYPE_IPV4: 838 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 839 break; 840 case IONIC_PKT_TYPE_IPV6: 841 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 842 break; 843 case IONIC_PKT_TYPE_IPV4_TCP: 844 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 845 RTE_PTYPE_L4_TCP; 846 break; 847 case IONIC_PKT_TYPE_IPV6_TCP: 848 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 849 RTE_PTYPE_L4_TCP; 850 break; 851 case IONIC_PKT_TYPE_IPV4_UDP: 852 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 853 RTE_PTYPE_L4_UDP; 854 break; 855 case IONIC_PKT_TYPE_IPV6_UDP: 856 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 857 RTE_PTYPE_L4_UDP; 858 break; 859 default: 860 { 861 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 862 struct rte_ether_hdr *); 863 uint16_t ether_type = eth_h->ether_type; 864 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 865 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 866 else 867 pkt_type = RTE_PTYPE_UNKNOWN; 868 break; 869 } 870 } 871 872 rxm->packet_type = pkt_type; 873 874 recv_args->rx_pkts[recv_args->nb_rx] = rxm; 875 recv_args->nb_rx++; 876 877 stats->packets++; 878 stats->bytes += rxm->pkt_len; 879 } 880 881 static void 882 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 883 struct rte_mbuf *mbuf) 884 { 885 struct ionic_rxq_desc *desc_base = q->base; 886 struct ionic_rxq_desc *old = &desc_base[q_desc_index]; 887 struct ionic_rxq_desc *new = &desc_base[q->head_idx]; 888 889 new->addr = old->addr; 890 new->len = old->len; 891 892 ionic_q_post(q, true, ionic_rx_clean, mbuf); 893 } 894 895 static int __rte_cold 896 ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) 897 { 898 struct ionic_queue *q = &rxq->q; 899 struct ionic_rxq_desc *desc_base = q->base; 900 struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base; 901 struct ionic_rxq_desc *desc; 902 struct ionic_rxq_sg_desc *sg_desc; 903 struct ionic_rxq_sg_elem *elem; 904 rte_iova_t dma_addr; 905 uint32_t i, j, nsegs, buf_size, size; 906 bool ring_doorbell; 907 908 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 909 RTE_PKTMBUF_HEADROOM); 910 911 /* Initialize software ring entries */ 912 for (i = ionic_q_space_avail(q); i; i--) { 913 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool); 914 struct rte_mbuf *prev_rxm_seg; 915 916 if (rxm == NULL) { 917 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 918 return -ENOMEM; 919 } 920 921 nsegs = (len + buf_size - 1) / buf_size; 922 923 desc = &desc_base[q->head_idx]; 924 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm)); 925 desc->addr = dma_addr; 926 desc->len = buf_size; 927 size = buf_size; 928 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 929 IONIC_RXQ_DESC_OPCODE_SIMPLE; 930 rxm->next = NULL; 931 932 prev_rxm_seg = rxm; 933 sg_desc = &sg_desc_base[q->head_idx]; 934 elem = sg_desc->elems; 935 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) { 936 struct rte_mbuf *rxm_seg; 937 rte_iova_t data_iova; 938 939 rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool); 940 if (rxm_seg == NULL) { 941 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 942 return -ENOMEM; 943 } 944 945 data_iova = rte_mbuf_data_iova(rxm_seg); 946 dma_addr = rte_cpu_to_le_64(data_iova); 947 elem->addr = dma_addr; 948 elem->len = buf_size; 949 size += buf_size; 950 elem++; 951 rxm_seg->next = NULL; 952 prev_rxm_seg->next = rxm_seg; 953 prev_rxm_seg = rxm_seg; 954 } 955 956 if (size < len) 957 IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)", 958 size, len); 959 960 ring_doorbell = ((q->head_idx + 1) & 961 IONIC_RX_RING_DOORBELL_STRIDE) == 0; 962 963 ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm); 964 } 965 966 return 0; 967 } 968 969 /* 970 * Start Receive Units for specified queue. 971 */ 972 int __rte_cold 973 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 974 { 975 uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 976 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; 977 struct ionic_qcq *rxq; 978 int err; 979 980 if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 981 IONIC_PRINT(DEBUG, "RX queue %u already started", 982 rx_queue_id); 983 return 0; 984 } 985 986 rxq = eth_dev->data->rx_queues[rx_queue_id]; 987 988 IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", 989 rx_queue_id, rxq->q.num_descs, frame_size); 990 991 if (!(rxq->flags & IONIC_QCQ_F_INITED)) { 992 err = ionic_lif_rxq_init(rxq); 993 if (err) 994 return err; 995 } else { 996 ionic_qcq_enable(rxq); 997 } 998 999 /* Allocate buffers for descriptor rings */ 1000 if (ionic_rx_fill(rxq, frame_size) != 0) { 1001 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d", 1002 rx_queue_id); 1003 return -1; 1004 } 1005 1006 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1007 1008 return 0; 1009 } 1010 1011 static inline void __rte_cold 1012 ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do, 1013 void *service_cb_arg) 1014 { 1015 struct ionic_queue *q = cq->bound_q; 1016 struct ionic_desc_info *q_desc_info; 1017 struct ionic_rxq_comp *cq_desc_base = cq->base; 1018 struct ionic_rxq_comp *cq_desc; 1019 bool more; 1020 uint32_t curr_q_tail_idx, curr_cq_tail_idx; 1021 uint32_t work_done = 0; 1022 1023 if (work_to_do == 0) 1024 return; 1025 1026 cq_desc = &cq_desc_base[cq->tail_idx]; 1027 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 1028 curr_cq_tail_idx = cq->tail_idx; 1029 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 1030 1031 if (cq->tail_idx == 0) 1032 cq->done_color = !cq->done_color; 1033 1034 /* Prefetch the next 4 descriptors */ 1035 if ((cq->tail_idx & 0x3) == 0) 1036 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 1037 1038 do { 1039 more = (q->tail_idx != cq_desc->comp_index); 1040 1041 q_desc_info = &q->info[q->tail_idx]; 1042 1043 curr_q_tail_idx = q->tail_idx; 1044 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1045 1046 /* Prefetch the next 4 descriptors */ 1047 if ((q->tail_idx & 0x3) == 0) 1048 /* q desc info */ 1049 rte_prefetch0(&q->info[q->tail_idx]); 1050 1051 ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx, 1052 q_desc_info->cb_arg, service_cb_arg); 1053 1054 } while (more); 1055 1056 if (++work_done == work_to_do) 1057 break; 1058 1059 cq_desc = &cq_desc_base[cq->tail_idx]; 1060 } 1061 } 1062 1063 /* 1064 * Stop Receive Units for specified queue. 1065 */ 1066 int __rte_cold 1067 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1068 { 1069 struct ionic_qcq *rxq; 1070 1071 IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); 1072 1073 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1074 1075 eth_dev->data->rx_queue_state[rx_queue_id] = 1076 RTE_ETH_QUEUE_STATE_STOPPED; 1077 1078 ionic_qcq_disable(rxq); 1079 1080 /* Flush */ 1081 ionic_rxq_service(&rxq->cq, -1, NULL); 1082 1083 return 0; 1084 } 1085 1086 uint16_t 1087 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1088 uint16_t nb_pkts) 1089 { 1090 struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; 1091 uint32_t frame_size = 1092 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 1093 struct ionic_cq *cq = &rxq->cq; 1094 struct ionic_rx_service service_cb_arg; 1095 1096 service_cb_arg.rx_pkts = rx_pkts; 1097 service_cb_arg.nb_pkts = nb_pkts; 1098 service_cb_arg.nb_rx = 0; 1099 1100 ionic_rxq_service(cq, nb_pkts, &service_cb_arg); 1101 1102 ionic_rx_fill(rxq, frame_size); 1103 1104 return service_cb_arg.nb_rx; 1105 } 1106