1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <stdint.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 #include <rte_log.h> 19 #include <rte_debug.h> 20 #include <rte_interrupts.h> 21 #include <rte_pci.h> 22 #include <rte_memory.h> 23 #include <rte_memzone.h> 24 #include <rte_launch.h> 25 #include <rte_eal.h> 26 #include <rte_per_lcore.h> 27 #include <rte_lcore.h> 28 #include <rte_atomic.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_mempool.h> 31 #include <rte_malloc.h> 32 #include <rte_mbuf.h> 33 #include <rte_ether.h> 34 #include <ethdev_driver.h> 35 #include <rte_prefetch.h> 36 #include <rte_udp.h> 37 #include <rte_tcp.h> 38 #include <rte_sctp.h> 39 #include <rte_string_fns.h> 40 #include <rte_errno.h> 41 #include <rte_ip.h> 42 #include <rte_net.h> 43 44 #include "ionic_logs.h" 45 #include "ionic_mac_api.h" 46 #include "ionic_ethdev.h" 47 #include "ionic_lif.h" 48 #include "ionic_rxtx.h" 49 50 #define IONIC_RX_RING_DOORBELL_STRIDE (32 - 1) 51 52 /********************************************************************* 53 * 54 * TX functions 55 * 56 **********************************************************************/ 57 58 void 59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 60 struct rte_eth_txq_info *qinfo) 61 { 62 struct ionic_qcq *txq = dev->data->tx_queues[queue_id]; 63 struct ionic_queue *q = &txq->q; 64 65 qinfo->nb_desc = q->num_descs; 66 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 67 qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED; 68 } 69 70 static __rte_always_inline void 71 ionic_tx_flush(struct ionic_cq *cq) 72 { 73 struct ionic_queue *q = cq->bound_q; 74 struct ionic_desc_info *q_desc_info; 75 struct rte_mbuf *txm, *next; 76 struct ionic_txq_comp *cq_desc_base = cq->base; 77 struct ionic_txq_comp *cq_desc; 78 u_int32_t comp_index = (u_int32_t)-1; 79 80 cq_desc = &cq_desc_base[cq->tail_idx]; 81 while (color_match(cq_desc->color, cq->done_color)) { 82 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 83 84 /* Prefetch the next 4 descriptors (not really useful here) */ 85 if ((cq->tail_idx & 0x3) == 0) 86 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 87 88 if (cq->tail_idx == 0) 89 cq->done_color = !cq->done_color; 90 91 comp_index = cq_desc->comp_index; 92 93 cq_desc = &cq_desc_base[cq->tail_idx]; 94 } 95 96 if (comp_index != (u_int32_t)-1) { 97 while (q->tail_idx != comp_index) { 98 q_desc_info = &q->info[q->tail_idx]; 99 100 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 101 102 /* Prefetch the next 4 descriptors */ 103 if ((q->tail_idx & 0x3) == 0) 104 /* q desc info */ 105 rte_prefetch0(&q->info[q->tail_idx]); 106 107 /* 108 * Note: you can just use rte_pktmbuf_free, 109 * but this loop is faster 110 */ 111 txm = q_desc_info->cb_arg; 112 while (txm != NULL) { 113 next = txm->next; 114 rte_pktmbuf_free_seg(txm); 115 txm = next; 116 } 117 } 118 } 119 } 120 121 void __rte_cold 122 ionic_dev_tx_queue_release(void *tx_queue) 123 { 124 struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; 125 126 IONIC_PRINT_CALL(); 127 128 ionic_lif_txq_deinit(txq); 129 130 ionic_qcq_free(txq); 131 } 132 133 int __rte_cold 134 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 135 { 136 struct ionic_qcq *txq; 137 138 IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); 139 140 txq = eth_dev->data->tx_queues[tx_queue_id]; 141 142 eth_dev->data->tx_queue_state[tx_queue_id] = 143 RTE_ETH_QUEUE_STATE_STOPPED; 144 145 /* 146 * Note: we should better post NOP Tx desc and wait for its completion 147 * before disabling Tx queue 148 */ 149 150 ionic_qcq_disable(txq); 151 152 ionic_tx_flush(&txq->cq); 153 154 return 0; 155 } 156 157 int __rte_cold 158 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, 159 uint16_t nb_desc, uint32_t socket_id, 160 const struct rte_eth_txconf *tx_conf) 161 { 162 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 163 struct ionic_qcq *txq; 164 uint64_t offloads; 165 int err; 166 167 if (tx_queue_id >= lif->ntxqcqs) { 168 IONIC_PRINT(DEBUG, "Queue index %u not available " 169 "(max %u queues)", 170 tx_queue_id, lif->ntxqcqs); 171 return -EINVAL; 172 } 173 174 offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads; 175 IONIC_PRINT(DEBUG, 176 "Configuring skt %u TX queue %u with %u buffers, offloads %jx", 177 socket_id, tx_queue_id, nb_desc, offloads); 178 179 /* Validate number of receive descriptors */ 180 if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC) 181 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 182 183 /* Free memory prior to re-allocation if needed... */ 184 if (eth_dev->data->tx_queues[tx_queue_id] != NULL) { 185 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id]; 186 ionic_dev_tx_queue_release(tx_queue); 187 eth_dev->data->tx_queues[tx_queue_id] = NULL; 188 } 189 190 eth_dev->data->tx_queue_state[tx_queue_id] = 191 RTE_ETH_QUEUE_STATE_STOPPED; 192 193 err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq); 194 if (err) { 195 IONIC_PRINT(DEBUG, "Queue allocation failure"); 196 return -EINVAL; 197 } 198 199 /* Do not start queue with rte_eth_dev_start() */ 200 if (tx_conf->tx_deferred_start) 201 txq->flags |= IONIC_QCQ_F_DEFERRED; 202 203 /* Convert the offload flags into queue flags */ 204 if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 205 txq->flags |= IONIC_QCQ_F_CSUM_L3; 206 if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM) 207 txq->flags |= IONIC_QCQ_F_CSUM_TCP; 208 if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM) 209 txq->flags |= IONIC_QCQ_F_CSUM_UDP; 210 211 eth_dev->data->tx_queues[tx_queue_id] = txq; 212 213 return 0; 214 } 215 216 /* 217 * Start Transmit Units for specified queue. 218 */ 219 int __rte_cold 220 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 221 { 222 uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; 223 struct ionic_qcq *txq; 224 int err; 225 226 if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 227 IONIC_PRINT(DEBUG, "TX queue %u already started", 228 tx_queue_id); 229 return 0; 230 } 231 232 txq = eth_dev->data->tx_queues[tx_queue_id]; 233 234 IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", 235 tx_queue_id, txq->q.num_descs); 236 237 if (!(txq->flags & IONIC_QCQ_F_INITED)) { 238 err = ionic_lif_txq_init(txq); 239 if (err) 240 return err; 241 } else { 242 ionic_qcq_enable(txq); 243 } 244 245 tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 246 247 return 0; 248 } 249 250 static void 251 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm) 252 { 253 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 254 char *l3_hdr = ((char *)eth_hdr) + txm->l2_len; 255 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 256 (l3_hdr + txm->l3_len); 257 258 if (txm->ol_flags & PKT_TX_IP_CKSUM) { 259 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 260 ipv4_hdr->hdr_checksum = 0; 261 tcp_hdr->cksum = 0; 262 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 263 } else { 264 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 265 tcp_hdr->cksum = 0; 266 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 267 } 268 } 269 270 static void 271 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm) 272 { 273 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 274 char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len + 275 txm->outer_l3_len + txm->l2_len; 276 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 277 (l3_hdr + txm->l3_len); 278 279 if (txm->ol_flags & PKT_TX_IPV4) { 280 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 281 ipv4_hdr->hdr_checksum = 0; 282 tcp_hdr->cksum = 0; 283 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 284 } else { 285 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 286 tcp_hdr->cksum = 0; 287 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 288 } 289 } 290 291 static void 292 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 293 struct rte_mbuf *txm, 294 rte_iova_t addr, uint8_t nsge, uint16_t len, 295 uint32_t hdrlen, uint32_t mss, 296 bool encap, 297 uint16_t vlan_tci, bool has_vlan, 298 bool start, bool done) 299 { 300 uint8_t flags = 0; 301 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 302 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 303 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 304 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 305 306 desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, 307 flags, nsge, addr); 308 desc->len = len; 309 desc->vlan_tci = vlan_tci; 310 desc->hdr_len = hdrlen; 311 desc->mss = mss; 312 313 ionic_q_post(q, done, NULL, done ? txm : NULL); 314 } 315 316 static struct ionic_txq_desc * 317 ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem) 318 { 319 struct ionic_txq_desc *desc_base = q->base; 320 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 321 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 322 struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; 323 324 *elem = sg_desc->elems; 325 return desc; 326 } 327 328 static int 329 ionic_tx_tso(struct ionic_qcq *txq, struct rte_mbuf *txm, 330 bool not_xmit_more) 331 { 332 struct ionic_queue *q = &txq->q; 333 struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); 334 struct ionic_txq_desc *desc; 335 struct ionic_txq_sg_elem *elem; 336 struct rte_mbuf *txm_seg; 337 rte_iova_t data_iova; 338 uint64_t desc_addr = 0, next_addr; 339 uint16_t desc_len = 0; 340 uint8_t desc_nsge; 341 uint32_t hdrlen; 342 uint32_t mss = txm->tso_segsz; 343 uint32_t frag_left = 0; 344 uint32_t left; 345 uint32_t seglen; 346 uint32_t len; 347 uint32_t offset = 0; 348 bool start, done; 349 bool encap; 350 bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT); 351 uint16_t vlan_tci = txm->vlan_tci; 352 uint64_t ol_flags = txm->ol_flags; 353 354 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 355 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 356 ((ol_flags & PKT_TX_OUTER_IPV4) || 357 (ol_flags & PKT_TX_OUTER_IPV6)); 358 359 /* Preload inner-most TCP csum field with IP pseudo hdr 360 * calculated with IP length set to zero. HW will later 361 * add in length to each TCP segment resulting from the TSO. 362 */ 363 364 if (encap) { 365 ionic_tx_tcp_inner_pseudo_csum(txm); 366 hdrlen = txm->outer_l2_len + txm->outer_l3_len + 367 txm->l2_len + txm->l3_len + txm->l4_len; 368 } else { 369 ionic_tx_tcp_pseudo_csum(txm); 370 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len; 371 } 372 373 seglen = hdrlen + mss; 374 left = txm->data_len; 375 data_iova = rte_mbuf_data_iova(txm); 376 377 desc = ionic_tx_tso_next(q, &elem); 378 start = true; 379 380 /* Chop data up into desc segments */ 381 382 while (left > 0) { 383 len = RTE_MIN(seglen, left); 384 frag_left = seglen - len; 385 desc_addr = rte_cpu_to_le_64(data_iova + offset); 386 desc_len = len; 387 desc_nsge = 0; 388 left -= len; 389 offset += len; 390 if (txm->nb_segs > 1 && frag_left > 0) 391 continue; 392 done = (txm->nb_segs == 1 && left == 0); 393 ionic_tx_tso_post(q, desc, txm, 394 desc_addr, desc_nsge, desc_len, 395 hdrlen, mss, 396 encap, 397 vlan_tci, has_vlan, 398 start, done && not_xmit_more); 399 desc = ionic_tx_tso_next(q, &elem); 400 start = false; 401 seglen = mss; 402 } 403 404 /* Chop frags into desc segments */ 405 406 txm_seg = txm->next; 407 while (txm_seg != NULL) { 408 offset = 0; 409 data_iova = rte_mbuf_data_iova(txm_seg); 410 left = txm_seg->data_len; 411 stats->frags++; 412 413 while (left > 0) { 414 next_addr = rte_cpu_to_le_64(data_iova + offset); 415 if (frag_left > 0) { 416 len = RTE_MIN(frag_left, left); 417 frag_left -= len; 418 elem->addr = next_addr; 419 elem->len = len; 420 elem++; 421 desc_nsge++; 422 } else { 423 len = RTE_MIN(mss, left); 424 frag_left = mss - len; 425 desc_addr = next_addr; 426 desc_len = len; 427 desc_nsge = 0; 428 } 429 left -= len; 430 offset += len; 431 if (txm_seg->next != NULL && frag_left > 0) 432 continue; 433 434 done = (txm_seg->next == NULL && left == 0); 435 ionic_tx_tso_post(q, desc, txm_seg, 436 desc_addr, desc_nsge, desc_len, 437 hdrlen, mss, 438 encap, 439 vlan_tci, has_vlan, 440 start, done && not_xmit_more); 441 desc = ionic_tx_tso_next(q, &elem); 442 start = false; 443 } 444 445 txm_seg = txm_seg->next; 446 } 447 448 stats->tso++; 449 450 return 0; 451 } 452 453 static __rte_always_inline int 454 ionic_tx(struct ionic_qcq *txq, struct rte_mbuf *txm, 455 bool not_xmit_more) 456 { 457 struct ionic_queue *q = &txq->q; 458 struct ionic_txq_desc *desc_base = q->base; 459 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 460 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 461 struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; 462 struct ionic_txq_sg_elem *elem = sg_desc->elems; 463 struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); 464 struct rte_mbuf *txm_seg; 465 bool encap; 466 bool has_vlan; 467 uint64_t ol_flags = txm->ol_flags; 468 uint64_t addr; 469 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 470 uint8_t flags = 0; 471 472 if ((ol_flags & PKT_TX_IP_CKSUM) && 473 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 474 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 475 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 476 } 477 478 if (((ol_flags & PKT_TX_TCP_CKSUM) && 479 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 480 ((ol_flags & PKT_TX_UDP_CKSUM) && 481 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 482 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 483 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 484 } 485 486 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 487 stats->no_csum++; 488 489 has_vlan = (ol_flags & PKT_TX_VLAN_PKT); 490 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 491 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 492 ((ol_flags & PKT_TX_OUTER_IPV4) || 493 (ol_flags & PKT_TX_OUTER_IPV6)); 494 495 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 496 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 497 498 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 499 500 desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 501 desc->len = txm->data_len; 502 desc->vlan_tci = txm->vlan_tci; 503 504 txm_seg = txm->next; 505 while (txm_seg != NULL) { 506 elem->len = txm_seg->data_len; 507 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); 508 stats->frags++; 509 elem++; 510 txm_seg = txm_seg->next; 511 } 512 513 ionic_q_post(q, not_xmit_more, NULL, txm); 514 515 return 0; 516 } 517 518 uint16_t 519 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 520 uint16_t nb_pkts) 521 { 522 struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue; 523 struct ionic_queue *q = &txq->q; 524 struct ionic_cq *cq = &txq->cq; 525 struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q); 526 uint32_t next_q_head_idx; 527 uint32_t bytes_tx = 0; 528 uint16_t nb_tx = 0; 529 int err; 530 bool last; 531 532 /* Cleaning old buffers */ 533 ionic_tx_flush(cq); 534 535 if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { 536 stats->stop += nb_pkts; 537 return 0; 538 } 539 540 while (nb_tx < nb_pkts) { 541 last = (nb_tx == (nb_pkts - 1)); 542 543 next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1); 544 if ((next_q_head_idx & 0x3) == 0) { 545 struct ionic_txq_desc *desc_base = q->base; 546 rte_prefetch0(&desc_base[next_q_head_idx]); 547 rte_prefetch0(&q->info[next_q_head_idx]); 548 } 549 550 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG) 551 err = ionic_tx_tso(txq, tx_pkts[nb_tx], last); 552 else 553 err = ionic_tx(txq, tx_pkts[nb_tx], last); 554 if (err) { 555 stats->drop += nb_pkts - nb_tx; 556 if (nb_tx > 0) 557 ionic_q_flush(q); 558 break; 559 } 560 561 bytes_tx += tx_pkts[nb_tx]->pkt_len; 562 nb_tx++; 563 } 564 565 stats->packets += nb_tx; 566 stats->bytes += bytes_tx; 567 568 return nb_tx; 569 } 570 571 /********************************************************************* 572 * 573 * TX prep functions 574 * 575 **********************************************************************/ 576 577 #define IONIC_TX_OFFLOAD_MASK ( \ 578 PKT_TX_IPV4 | \ 579 PKT_TX_IPV6 | \ 580 PKT_TX_VLAN | \ 581 PKT_TX_IP_CKSUM | \ 582 PKT_TX_TCP_SEG | \ 583 PKT_TX_L4_MASK) 584 585 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \ 586 (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) 587 588 uint16_t 589 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 590 uint16_t nb_pkts) 591 { 592 struct rte_mbuf *txm; 593 uint64_t offloads; 594 int i = 0; 595 596 for (i = 0; i < nb_pkts; i++) { 597 txm = tx_pkts[i]; 598 599 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) { 600 rte_errno = -EINVAL; 601 break; 602 } 603 604 offloads = txm->ol_flags; 605 606 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) { 607 rte_errno = -ENOTSUP; 608 break; 609 } 610 } 611 612 return i; 613 } 614 615 /********************************************************************* 616 * 617 * RX functions 618 * 619 **********************************************************************/ 620 621 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 622 struct rte_mbuf *mbuf); 623 624 void 625 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 626 struct rte_eth_rxq_info *qinfo) 627 { 628 struct ionic_qcq *rxq = dev->data->rx_queues[queue_id]; 629 struct ionic_queue *q = &rxq->q; 630 631 qinfo->mp = rxq->mb_pool; 632 qinfo->scattered_rx = dev->data->scattered_rx; 633 qinfo->nb_desc = q->num_descs; 634 qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED; 635 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 636 } 637 638 static void __rte_cold 639 ionic_rx_empty(struct ionic_queue *q) 640 { 641 struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); 642 struct ionic_desc_info *cur; 643 struct rte_mbuf *mbuf; 644 645 while (q->tail_idx != q->head_idx) { 646 cur = &q->info[q->tail_idx]; 647 mbuf = cur->cb_arg; 648 rte_mempool_put(rxq->mb_pool, mbuf); 649 650 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 651 } 652 } 653 654 void __rte_cold 655 ionic_dev_rx_queue_release(void *rx_queue) 656 { 657 struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; 658 659 IONIC_PRINT_CALL(); 660 661 ionic_rx_empty(&rxq->q); 662 663 ionic_lif_rxq_deinit(rxq); 664 665 ionic_qcq_free(rxq); 666 } 667 668 int __rte_cold 669 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, 670 uint16_t rx_queue_id, 671 uint16_t nb_desc, 672 uint32_t socket_id, 673 const struct rte_eth_rxconf *rx_conf, 674 struct rte_mempool *mp) 675 { 676 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 677 struct ionic_qcq *rxq; 678 uint64_t offloads; 679 int err; 680 681 if (rx_queue_id >= lif->nrxqcqs) { 682 IONIC_PRINT(ERR, 683 "Queue index %u not available (max %u queues)", 684 rx_queue_id, lif->nrxqcqs); 685 return -EINVAL; 686 } 687 688 offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads; 689 IONIC_PRINT(DEBUG, 690 "Configuring skt %u RX queue %u with %u buffers, offloads %jx", 691 socket_id, rx_queue_id, nb_desc, offloads); 692 693 if (!rx_conf->rx_drop_en) 694 IONIC_PRINT(WARNING, "No-drop mode is not supported"); 695 696 /* Validate number of receive descriptors */ 697 if (!rte_is_power_of_2(nb_desc) || 698 nb_desc < IONIC_MIN_RING_DESC || 699 nb_desc > IONIC_MAX_RING_DESC) { 700 IONIC_PRINT(ERR, 701 "Bad descriptor count (%u) for queue %u (min: %u)", 702 nb_desc, rx_queue_id, IONIC_MIN_RING_DESC); 703 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 704 } 705 706 /* Free memory prior to re-allocation if needed... */ 707 if (eth_dev->data->rx_queues[rx_queue_id] != NULL) { 708 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id]; 709 ionic_dev_rx_queue_release(rx_queue); 710 eth_dev->data->rx_queues[rx_queue_id] = NULL; 711 } 712 713 eth_dev->data->rx_queue_state[rx_queue_id] = 714 RTE_ETH_QUEUE_STATE_STOPPED; 715 716 err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq); 717 if (err) { 718 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); 719 return -EINVAL; 720 } 721 722 rxq->mb_pool = mp; 723 724 /* 725 * Note: the interface does not currently support 726 * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN 727 * when the adapter will be able to keep the CRC and subtract 728 * it to the length for all received packets: 729 * if (eth_dev->data->dev_conf.rxmode.offloads & 730 * DEV_RX_OFFLOAD_KEEP_CRC) 731 * rxq->crc_len = ETHER_CRC_LEN; 732 */ 733 734 /* Do not start queue with rte_eth_dev_start() */ 735 if (rx_conf->rx_deferred_start) 736 rxq->flags |= IONIC_QCQ_F_DEFERRED; 737 738 eth_dev->data->rx_queues[rx_queue_id] = rxq; 739 740 return 0; 741 } 742 743 static __rte_always_inline void 744 ionic_rx_clean(struct ionic_queue *q, 745 uint32_t q_desc_index, uint32_t cq_desc_index, 746 void *cb_arg, void *service_cb_arg) 747 { 748 struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base; 749 struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 750 struct rte_mbuf *rxm = cb_arg; 751 struct rte_mbuf *rxm_seg; 752 struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q); 753 uint32_t max_frame_size = 754 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 755 uint64_t pkt_flags = 0; 756 uint32_t pkt_type; 757 struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q); 758 struct ionic_rx_service *recv_args = (struct ionic_rx_service *) 759 service_cb_arg; 760 uint32_t buf_size = (uint16_t) 761 (rte_pktmbuf_data_room_size(rxq->mb_pool) - 762 RTE_PKTMBUF_HEADROOM); 763 uint32_t left; 764 765 if (!recv_args) { 766 stats->no_cb_arg++; 767 /* Flush */ 768 rte_pktmbuf_free(rxm); 769 /* 770 * Note: rte_mempool_put is faster with no segs 771 * rte_mempool_put(rxq->mb_pool, rxm); 772 */ 773 return; 774 } 775 776 if (cq_desc->status) { 777 stats->bad_cq_status++; 778 ionic_rx_recycle(q, q_desc_index, rxm); 779 return; 780 } 781 782 if (recv_args->nb_rx >= recv_args->nb_pkts) { 783 stats->no_room++; 784 ionic_rx_recycle(q, q_desc_index, rxm); 785 return; 786 } 787 788 if (cq_desc->len > max_frame_size || 789 cq_desc->len == 0) { 790 stats->bad_len++; 791 ionic_rx_recycle(q, q_desc_index, rxm); 792 return; 793 } 794 795 rxm->data_off = RTE_PKTMBUF_HEADROOM; 796 rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); 797 rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ 798 rxm->pkt_len = cq_desc->len; 799 rxm->port = rxq->lif->port_id; 800 801 left = cq_desc->len; 802 803 rxm->data_len = RTE_MIN(buf_size, left); 804 left -= rxm->data_len; 805 806 rxm_seg = rxm->next; 807 while (rxm_seg && left) { 808 rxm_seg->data_len = RTE_MIN(buf_size, left); 809 left -= rxm_seg->data_len; 810 811 rxm_seg = rxm_seg->next; 812 rxm->nb_segs++; 813 } 814 815 /* RSS */ 816 pkt_flags |= PKT_RX_RSS_HASH; 817 rxm->hash.rss = cq_desc->rss_hash; 818 819 /* Vlan Strip */ 820 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 821 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 822 rxm->vlan_tci = cq_desc->vlan_tci; 823 } 824 825 /* Checksum */ 826 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 827 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK) 828 pkt_flags |= PKT_RX_IP_CKSUM_GOOD; 829 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD) 830 pkt_flags |= PKT_RX_IP_CKSUM_BAD; 831 832 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) || 833 (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK)) 834 pkt_flags |= PKT_RX_L4_CKSUM_GOOD; 835 else if ((cq_desc->csum_flags & 836 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 837 (cq_desc->csum_flags & 838 IONIC_RXQ_COMP_CSUM_F_UDP_BAD)) 839 pkt_flags |= PKT_RX_L4_CKSUM_BAD; 840 } 841 842 rxm->ol_flags = pkt_flags; 843 844 /* Packet Type */ 845 switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 846 case IONIC_PKT_TYPE_IPV4: 847 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 848 break; 849 case IONIC_PKT_TYPE_IPV6: 850 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 851 break; 852 case IONIC_PKT_TYPE_IPV4_TCP: 853 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 854 RTE_PTYPE_L4_TCP; 855 break; 856 case IONIC_PKT_TYPE_IPV6_TCP: 857 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 858 RTE_PTYPE_L4_TCP; 859 break; 860 case IONIC_PKT_TYPE_IPV4_UDP: 861 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 862 RTE_PTYPE_L4_UDP; 863 break; 864 case IONIC_PKT_TYPE_IPV6_UDP: 865 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 866 RTE_PTYPE_L4_UDP; 867 break; 868 default: 869 { 870 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 871 struct rte_ether_hdr *); 872 uint16_t ether_type = eth_h->ether_type; 873 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 874 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 875 else 876 pkt_type = RTE_PTYPE_UNKNOWN; 877 break; 878 } 879 } 880 881 rxm->packet_type = pkt_type; 882 883 recv_args->rx_pkts[recv_args->nb_rx] = rxm; 884 recv_args->nb_rx++; 885 886 stats->packets++; 887 stats->bytes += rxm->pkt_len; 888 } 889 890 static void 891 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 892 struct rte_mbuf *mbuf) 893 { 894 struct ionic_rxq_desc *desc_base = q->base; 895 struct ionic_rxq_desc *old = &desc_base[q_desc_index]; 896 struct ionic_rxq_desc *new = &desc_base[q->head_idx]; 897 898 new->addr = old->addr; 899 new->len = old->len; 900 901 ionic_q_post(q, true, ionic_rx_clean, mbuf); 902 } 903 904 static __rte_always_inline int 905 ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len) 906 { 907 struct ionic_queue *q = &rxq->q; 908 struct ionic_rxq_desc *desc_base = q->base; 909 struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base; 910 struct ionic_rxq_desc *desc; 911 struct ionic_rxq_sg_desc *sg_desc; 912 struct ionic_rxq_sg_elem *elem; 913 rte_iova_t dma_addr; 914 uint32_t i, j, nsegs, buf_size, size; 915 bool ring_doorbell; 916 917 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 918 RTE_PKTMBUF_HEADROOM); 919 920 /* Initialize software ring entries */ 921 for (i = ionic_q_space_avail(q); i; i--) { 922 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool); 923 struct rte_mbuf *prev_rxm_seg; 924 925 if (rxm == NULL) { 926 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 927 return -ENOMEM; 928 } 929 930 nsegs = (len + buf_size - 1) / buf_size; 931 932 desc = &desc_base[q->head_idx]; 933 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm)); 934 desc->addr = dma_addr; 935 desc->len = buf_size; 936 size = buf_size; 937 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 938 IONIC_RXQ_DESC_OPCODE_SIMPLE; 939 rxm->next = NULL; 940 941 prev_rxm_seg = rxm; 942 sg_desc = &sg_desc_base[q->head_idx]; 943 elem = sg_desc->elems; 944 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) { 945 struct rte_mbuf *rxm_seg; 946 rte_iova_t data_iova; 947 948 rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool); 949 if (rxm_seg == NULL) { 950 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 951 return -ENOMEM; 952 } 953 954 data_iova = rte_mbuf_data_iova(rxm_seg); 955 dma_addr = rte_cpu_to_le_64(data_iova); 956 elem->addr = dma_addr; 957 elem->len = buf_size; 958 size += buf_size; 959 elem++; 960 rxm_seg->next = NULL; 961 prev_rxm_seg->next = rxm_seg; 962 prev_rxm_seg = rxm_seg; 963 } 964 965 if (size < len) 966 IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)", 967 size, len); 968 969 ring_doorbell = ((q->head_idx + 1) & 970 IONIC_RX_RING_DOORBELL_STRIDE) == 0; 971 972 ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm); 973 } 974 975 return 0; 976 } 977 978 /* 979 * Start Receive Units for specified queue. 980 */ 981 int __rte_cold 982 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 983 { 984 uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 985 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; 986 struct ionic_qcq *rxq; 987 int err; 988 989 if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 990 IONIC_PRINT(DEBUG, "RX queue %u already started", 991 rx_queue_id); 992 return 0; 993 } 994 995 rxq = eth_dev->data->rx_queues[rx_queue_id]; 996 997 IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", 998 rx_queue_id, rxq->q.num_descs, frame_size); 999 1000 if (!(rxq->flags & IONIC_QCQ_F_INITED)) { 1001 err = ionic_lif_rxq_init(rxq); 1002 if (err) 1003 return err; 1004 } else { 1005 ionic_qcq_enable(rxq); 1006 } 1007 1008 /* Allocate buffers for descriptor rings */ 1009 if (ionic_rx_fill(rxq, frame_size) != 0) { 1010 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d", 1011 rx_queue_id); 1012 return -1; 1013 } 1014 1015 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1016 1017 return 0; 1018 } 1019 1020 static __rte_always_inline void 1021 ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do, 1022 void *service_cb_arg) 1023 { 1024 struct ionic_queue *q = cq->bound_q; 1025 struct ionic_desc_info *q_desc_info; 1026 struct ionic_rxq_comp *cq_desc_base = cq->base; 1027 struct ionic_rxq_comp *cq_desc; 1028 bool more; 1029 uint32_t curr_q_tail_idx, curr_cq_tail_idx; 1030 uint32_t work_done = 0; 1031 1032 if (work_to_do == 0) 1033 return; 1034 1035 cq_desc = &cq_desc_base[cq->tail_idx]; 1036 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 1037 curr_cq_tail_idx = cq->tail_idx; 1038 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 1039 1040 if (cq->tail_idx == 0) 1041 cq->done_color = !cq->done_color; 1042 1043 /* Prefetch the next 4 descriptors */ 1044 if ((cq->tail_idx & 0x3) == 0) 1045 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 1046 1047 do { 1048 more = (q->tail_idx != cq_desc->comp_index); 1049 1050 q_desc_info = &q->info[q->tail_idx]; 1051 1052 curr_q_tail_idx = q->tail_idx; 1053 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 1054 1055 /* Prefetch the next 4 descriptors */ 1056 if ((q->tail_idx & 0x3) == 0) 1057 /* q desc info */ 1058 rte_prefetch0(&q->info[q->tail_idx]); 1059 1060 ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx, 1061 q_desc_info->cb_arg, service_cb_arg); 1062 1063 } while (more); 1064 1065 if (++work_done == work_to_do) 1066 break; 1067 1068 cq_desc = &cq_desc_base[cq->tail_idx]; 1069 } 1070 } 1071 1072 /* 1073 * Stop Receive Units for specified queue. 1074 */ 1075 int __rte_cold 1076 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1077 { 1078 struct ionic_qcq *rxq; 1079 1080 IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); 1081 1082 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1083 1084 eth_dev->data->rx_queue_state[rx_queue_id] = 1085 RTE_ETH_QUEUE_STATE_STOPPED; 1086 1087 ionic_qcq_disable(rxq); 1088 1089 /* Flush */ 1090 ionic_rxq_service(&rxq->cq, -1, NULL); 1091 1092 return 0; 1093 } 1094 1095 uint16_t 1096 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1097 uint16_t nb_pkts) 1098 { 1099 struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue; 1100 uint32_t frame_size = 1101 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 1102 struct ionic_cq *cq = &rxq->cq; 1103 struct ionic_rx_service service_cb_arg; 1104 1105 service_cb_arg.rx_pkts = rx_pkts; 1106 service_cb_arg.nb_pkts = nb_pkts; 1107 service_cb_arg.nb_rx = 0; 1108 1109 ionic_rxq_service(cq, nb_pkts, &service_cb_arg); 1110 1111 ionic_rx_fill(rxq, frame_size); 1112 1113 return service_cb_arg.nb_rx; 1114 } 1115