1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <stdint.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 #include <rte_log.h> 19 #include <rte_debug.h> 20 #include <rte_interrupts.h> 21 #include <rte_pci.h> 22 #include <rte_memory.h> 23 #include <rte_memzone.h> 24 #include <rte_launch.h> 25 #include <rte_eal.h> 26 #include <rte_per_lcore.h> 27 #include <rte_lcore.h> 28 #include <rte_atomic.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_mempool.h> 31 #include <rte_malloc.h> 32 #include <rte_mbuf.h> 33 #include <rte_ether.h> 34 #include <ethdev_driver.h> 35 #include <rte_prefetch.h> 36 #include <rte_udp.h> 37 #include <rte_tcp.h> 38 #include <rte_sctp.h> 39 #include <rte_string_fns.h> 40 #include <rte_errno.h> 41 #include <rte_ip.h> 42 #include <rte_net.h> 43 44 #include "ionic_logs.h" 45 #include "ionic_mac_api.h" 46 #include "ionic_ethdev.h" 47 #include "ionic_lif.h" 48 #include "ionic_rxtx.h" 49 50 #define IONIC_RX_RING_DOORBELL_STRIDE (32 - 1) 51 52 /********************************************************************* 53 * 54 * TX functions 55 * 56 **********************************************************************/ 57 58 void 59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 60 struct rte_eth_txq_info *qinfo) 61 { 62 struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id]; 63 struct ionic_queue *q = &txq->qcq.q; 64 65 qinfo->nb_desc = q->num_descs; 66 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 67 qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED; 68 } 69 70 static __rte_always_inline void 71 ionic_tx_flush(struct ionic_tx_qcq *txq) 72 { 73 struct ionic_cq *cq = &txq->qcq.cq; 74 struct ionic_queue *q = &txq->qcq.q; 75 struct rte_mbuf *txm, *next; 76 struct ionic_txq_comp *cq_desc_base = cq->base; 77 struct ionic_txq_comp *cq_desc; 78 void **info; 79 u_int32_t comp_index = (u_int32_t)-1; 80 81 cq_desc = &cq_desc_base[cq->tail_idx]; 82 while (color_match(cq_desc->color, cq->done_color)) { 83 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 84 85 /* Prefetch the next 4 descriptors (not really useful here) */ 86 if ((cq->tail_idx & 0x3) == 0) 87 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 88 89 if (cq->tail_idx == 0) 90 cq->done_color = !cq->done_color; 91 92 comp_index = cq_desc->comp_index; 93 94 cq_desc = &cq_desc_base[cq->tail_idx]; 95 } 96 97 if (comp_index != (u_int32_t)-1) { 98 while (q->tail_idx != comp_index) { 99 info = IONIC_INFO_PTR(q, q->tail_idx); 100 101 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 102 103 /* Prefetch the next 4 descriptors */ 104 if ((q->tail_idx & 0x3) == 0) 105 /* q desc info */ 106 rte_prefetch0(&q->info[q->tail_idx]); 107 108 /* 109 * Note: you can just use rte_pktmbuf_free, 110 * but this loop is faster 111 */ 112 txm = info[0]; 113 while (txm != NULL) { 114 next = txm->next; 115 rte_pktmbuf_free_seg(txm); 116 txm = next; 117 } 118 } 119 } 120 } 121 122 void __rte_cold 123 ionic_dev_tx_queue_release(void *tx_queue) 124 { 125 struct ionic_tx_qcq *txq = tx_queue; 126 struct ionic_tx_stats *stats = &txq->stats; 127 128 IONIC_PRINT_CALL(); 129 130 IONIC_PRINT(DEBUG, "TX queue %u pkts %ju tso %ju", 131 txq->qcq.q.index, stats->packets, stats->tso); 132 133 ionic_lif_txq_deinit(txq); 134 135 ionic_qcq_free(&txq->qcq); 136 } 137 138 int __rte_cold 139 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 140 { 141 struct ionic_tx_qcq *txq; 142 143 IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); 144 145 txq = eth_dev->data->tx_queues[tx_queue_id]; 146 147 eth_dev->data->tx_queue_state[tx_queue_id] = 148 RTE_ETH_QUEUE_STATE_STOPPED; 149 150 /* 151 * Note: we should better post NOP Tx desc and wait for its completion 152 * before disabling Tx queue 153 */ 154 155 ionic_qcq_disable(&txq->qcq); 156 157 ionic_tx_flush(txq); 158 159 return 0; 160 } 161 162 int __rte_cold 163 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, 164 uint16_t nb_desc, uint32_t socket_id, 165 const struct rte_eth_txconf *tx_conf) 166 { 167 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 168 struct ionic_tx_qcq *txq; 169 uint64_t offloads; 170 int err; 171 172 if (tx_queue_id >= lif->ntxqcqs) { 173 IONIC_PRINT(DEBUG, "Queue index %u not available " 174 "(max %u queues)", 175 tx_queue_id, lif->ntxqcqs); 176 return -EINVAL; 177 } 178 179 offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads; 180 IONIC_PRINT(DEBUG, 181 "Configuring skt %u TX queue %u with %u buffers, offloads %jx", 182 socket_id, tx_queue_id, nb_desc, offloads); 183 184 /* Validate number of receive descriptors */ 185 if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC) 186 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 187 188 /* Free memory prior to re-allocation if needed... */ 189 if (eth_dev->data->tx_queues[tx_queue_id] != NULL) { 190 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id]; 191 ionic_dev_tx_queue_release(tx_queue); 192 eth_dev->data->tx_queues[tx_queue_id] = NULL; 193 } 194 195 eth_dev->data->tx_queue_state[tx_queue_id] = 196 RTE_ETH_QUEUE_STATE_STOPPED; 197 198 err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq); 199 if (err) { 200 IONIC_PRINT(DEBUG, "Queue allocation failure"); 201 return -EINVAL; 202 } 203 204 /* Do not start queue with rte_eth_dev_start() */ 205 if (tx_conf->tx_deferred_start) 206 txq->flags |= IONIC_QCQ_F_DEFERRED; 207 208 /* Convert the offload flags into queue flags */ 209 if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 210 txq->flags |= IONIC_QCQ_F_CSUM_L3; 211 if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM) 212 txq->flags |= IONIC_QCQ_F_CSUM_TCP; 213 if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM) 214 txq->flags |= IONIC_QCQ_F_CSUM_UDP; 215 216 eth_dev->data->tx_queues[tx_queue_id] = txq; 217 218 return 0; 219 } 220 221 /* 222 * Start Transmit Units for specified queue. 223 */ 224 int __rte_cold 225 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 226 { 227 uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; 228 struct ionic_tx_qcq *txq; 229 int err; 230 231 if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 232 IONIC_PRINT(DEBUG, "TX queue %u already started", 233 tx_queue_id); 234 return 0; 235 } 236 237 txq = eth_dev->data->tx_queues[tx_queue_id]; 238 239 IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", 240 tx_queue_id, txq->qcq.q.num_descs); 241 242 if (!(txq->flags & IONIC_QCQ_F_INITED)) { 243 err = ionic_lif_txq_init(txq); 244 if (err) 245 return err; 246 } else { 247 ionic_qcq_enable(&txq->qcq); 248 } 249 250 tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 251 252 return 0; 253 } 254 255 static void 256 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm) 257 { 258 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 259 char *l3_hdr = ((char *)eth_hdr) + txm->l2_len; 260 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 261 (l3_hdr + txm->l3_len); 262 263 if (txm->ol_flags & PKT_TX_IP_CKSUM) { 264 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 265 ipv4_hdr->hdr_checksum = 0; 266 tcp_hdr->cksum = 0; 267 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 268 } else { 269 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 270 tcp_hdr->cksum = 0; 271 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 272 } 273 } 274 275 static void 276 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm) 277 { 278 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 279 char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len + 280 txm->outer_l3_len + txm->l2_len; 281 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 282 (l3_hdr + txm->l3_len); 283 284 if (txm->ol_flags & PKT_TX_IPV4) { 285 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 286 ipv4_hdr->hdr_checksum = 0; 287 tcp_hdr->cksum = 0; 288 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 289 } else { 290 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 291 tcp_hdr->cksum = 0; 292 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 293 } 294 } 295 296 static void 297 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 298 struct rte_mbuf *txm, 299 rte_iova_t addr, uint8_t nsge, uint16_t len, 300 uint32_t hdrlen, uint32_t mss, 301 bool encap, 302 uint16_t vlan_tci, bool has_vlan, 303 bool start, bool done) 304 { 305 void **info; 306 uint8_t flags = 0; 307 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 308 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 309 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 310 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 311 312 desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, 313 flags, nsge, addr); 314 desc->len = len; 315 desc->vlan_tci = vlan_tci; 316 desc->hdr_len = hdrlen; 317 desc->mss = mss; 318 319 if (done) { 320 info = IONIC_INFO_PTR(q, q->head_idx); 321 info[0] = txm; 322 } 323 324 q->head_idx = Q_NEXT_TO_POST(q, 1); 325 326 if (done) 327 ionic_q_flush(q); 328 } 329 330 static struct ionic_txq_desc * 331 ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) 332 { 333 struct ionic_queue *q = &txq->qcq.q; 334 struct ionic_txq_desc *desc_base = q->base; 335 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 336 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 337 struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; 338 339 *elem = sg_desc->elems; 340 return desc; 341 } 342 343 static int 344 ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, 345 bool not_xmit_more) 346 { 347 struct ionic_queue *q = &txq->qcq.q; 348 struct ionic_tx_stats *stats = &txq->stats; 349 struct ionic_txq_desc *desc; 350 struct ionic_txq_sg_elem *elem; 351 struct rte_mbuf *txm_seg; 352 rte_iova_t data_iova; 353 uint64_t desc_addr = 0, next_addr; 354 uint16_t desc_len = 0; 355 uint8_t desc_nsge; 356 uint32_t hdrlen; 357 uint32_t mss = txm->tso_segsz; 358 uint32_t frag_left = 0; 359 uint32_t left; 360 uint32_t seglen; 361 uint32_t len; 362 uint32_t offset = 0; 363 bool start, done; 364 bool encap; 365 bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT); 366 uint16_t vlan_tci = txm->vlan_tci; 367 uint64_t ol_flags = txm->ol_flags; 368 369 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 370 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 371 ((ol_flags & PKT_TX_OUTER_IPV4) || 372 (ol_flags & PKT_TX_OUTER_IPV6)); 373 374 /* Preload inner-most TCP csum field with IP pseudo hdr 375 * calculated with IP length set to zero. HW will later 376 * add in length to each TCP segment resulting from the TSO. 377 */ 378 379 if (encap) { 380 ionic_tx_tcp_inner_pseudo_csum(txm); 381 hdrlen = txm->outer_l2_len + txm->outer_l3_len + 382 txm->l2_len + txm->l3_len + txm->l4_len; 383 } else { 384 ionic_tx_tcp_pseudo_csum(txm); 385 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len; 386 } 387 388 seglen = hdrlen + mss; 389 left = txm->data_len; 390 data_iova = rte_mbuf_data_iova(txm); 391 392 desc = ionic_tx_tso_next(txq, &elem); 393 start = true; 394 395 /* Chop data up into desc segments */ 396 397 while (left > 0) { 398 len = RTE_MIN(seglen, left); 399 frag_left = seglen - len; 400 desc_addr = rte_cpu_to_le_64(data_iova + offset); 401 desc_len = len; 402 desc_nsge = 0; 403 left -= len; 404 offset += len; 405 if (txm->nb_segs > 1 && frag_left > 0) 406 continue; 407 done = (txm->nb_segs == 1 && left == 0); 408 ionic_tx_tso_post(q, desc, txm, 409 desc_addr, desc_nsge, desc_len, 410 hdrlen, mss, 411 encap, 412 vlan_tci, has_vlan, 413 start, done && not_xmit_more); 414 desc = ionic_tx_tso_next(txq, &elem); 415 start = false; 416 seglen = mss; 417 } 418 419 /* Chop frags into desc segments */ 420 421 txm_seg = txm->next; 422 while (txm_seg != NULL) { 423 offset = 0; 424 data_iova = rte_mbuf_data_iova(txm_seg); 425 left = txm_seg->data_len; 426 427 while (left > 0) { 428 next_addr = rte_cpu_to_le_64(data_iova + offset); 429 if (frag_left > 0) { 430 len = RTE_MIN(frag_left, left); 431 frag_left -= len; 432 elem->addr = next_addr; 433 elem->len = len; 434 elem++; 435 desc_nsge++; 436 } else { 437 len = RTE_MIN(mss, left); 438 frag_left = mss - len; 439 desc_addr = next_addr; 440 desc_len = len; 441 desc_nsge = 0; 442 } 443 left -= len; 444 offset += len; 445 if (txm_seg->next != NULL && frag_left > 0) 446 continue; 447 448 done = (txm_seg->next == NULL && left == 0); 449 ionic_tx_tso_post(q, desc, txm_seg, 450 desc_addr, desc_nsge, desc_len, 451 hdrlen, mss, 452 encap, 453 vlan_tci, has_vlan, 454 start, done && not_xmit_more); 455 desc = ionic_tx_tso_next(txq, &elem); 456 start = false; 457 } 458 459 txm_seg = txm_seg->next; 460 } 461 462 stats->tso++; 463 464 return 0; 465 } 466 467 static __rte_always_inline int 468 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, 469 bool not_xmit_more) 470 { 471 struct ionic_queue *q = &txq->qcq.q; 472 struct ionic_txq_desc *desc, *desc_base = q->base; 473 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 474 struct ionic_txq_sg_elem *elem; 475 struct ionic_tx_stats *stats = &txq->stats; 476 struct rte_mbuf *txm_seg; 477 void **info; 478 bool encap; 479 bool has_vlan; 480 uint64_t ol_flags = txm->ol_flags; 481 uint64_t addr; 482 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 483 uint8_t flags = 0; 484 485 desc = &desc_base[q->head_idx]; 486 info = IONIC_INFO_PTR(q, q->head_idx); 487 488 if ((ol_flags & PKT_TX_IP_CKSUM) && 489 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 490 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 491 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 492 } 493 494 if (((ol_flags & PKT_TX_TCP_CKSUM) && 495 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 496 ((ol_flags & PKT_TX_UDP_CKSUM) && 497 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 498 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 499 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 500 } 501 502 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 503 stats->no_csum++; 504 505 has_vlan = (ol_flags & PKT_TX_VLAN_PKT); 506 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 507 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 508 ((ol_flags & PKT_TX_OUTER_IPV4) || 509 (ol_flags & PKT_TX_OUTER_IPV6)); 510 511 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 512 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 513 514 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 515 516 desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 517 desc->len = txm->data_len; 518 desc->vlan_tci = txm->vlan_tci; 519 520 info[0] = txm; 521 522 elem = sg_desc_base[q->head_idx].elems; 523 524 txm_seg = txm->next; 525 while (txm_seg != NULL) { 526 elem->len = txm_seg->data_len; 527 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); 528 elem++; 529 txm_seg = txm_seg->next; 530 } 531 532 q->head_idx = Q_NEXT_TO_POST(q, 1); 533 534 if (not_xmit_more) 535 ionic_q_flush(q); 536 537 return 0; 538 } 539 540 uint16_t 541 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 542 uint16_t nb_pkts) 543 { 544 struct ionic_tx_qcq *txq = tx_queue; 545 struct ionic_queue *q = &txq->qcq.q; 546 struct ionic_tx_stats *stats = &txq->stats; 547 uint32_t next_q_head_idx; 548 uint32_t bytes_tx = 0; 549 uint16_t nb_tx = 0; 550 int err; 551 bool last; 552 553 /* Cleaning old buffers */ 554 ionic_tx_flush(txq); 555 556 if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { 557 stats->stop += nb_pkts; 558 return 0; 559 } 560 561 while (nb_tx < nb_pkts) { 562 last = (nb_tx == (nb_pkts - 1)); 563 564 next_q_head_idx = Q_NEXT_TO_POST(q, 1); 565 if ((next_q_head_idx & 0x3) == 0) { 566 struct ionic_txq_desc *desc_base = q->base; 567 rte_prefetch0(&desc_base[next_q_head_idx]); 568 rte_prefetch0(&q->info[next_q_head_idx]); 569 } 570 571 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG) 572 err = ionic_tx_tso(txq, tx_pkts[nb_tx], last); 573 else 574 err = ionic_tx(txq, tx_pkts[nb_tx], last); 575 if (err) { 576 stats->drop += nb_pkts - nb_tx; 577 if (nb_tx > 0) 578 ionic_q_flush(q); 579 break; 580 } 581 582 bytes_tx += tx_pkts[nb_tx]->pkt_len; 583 nb_tx++; 584 } 585 586 stats->packets += nb_tx; 587 stats->bytes += bytes_tx; 588 589 return nb_tx; 590 } 591 592 /********************************************************************* 593 * 594 * TX prep functions 595 * 596 **********************************************************************/ 597 598 #define IONIC_TX_OFFLOAD_MASK ( \ 599 PKT_TX_IPV4 | \ 600 PKT_TX_IPV6 | \ 601 PKT_TX_VLAN | \ 602 PKT_TX_IP_CKSUM | \ 603 PKT_TX_TCP_SEG | \ 604 PKT_TX_L4_MASK) 605 606 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \ 607 (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) 608 609 uint16_t 610 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 611 uint16_t nb_pkts) 612 { 613 struct rte_mbuf *txm; 614 uint64_t offloads; 615 int i = 0; 616 617 for (i = 0; i < nb_pkts; i++) { 618 txm = tx_pkts[i]; 619 620 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) { 621 rte_errno = -EINVAL; 622 break; 623 } 624 625 offloads = txm->ol_flags; 626 627 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) { 628 rte_errno = -ENOTSUP; 629 break; 630 } 631 } 632 633 return i; 634 } 635 636 /********************************************************************* 637 * 638 * RX functions 639 * 640 **********************************************************************/ 641 642 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 643 struct rte_mbuf *mbuf); 644 645 void 646 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 647 struct rte_eth_rxq_info *qinfo) 648 { 649 struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id]; 650 struct ionic_queue *q = &rxq->qcq.q; 651 652 qinfo->mp = rxq->mb_pool; 653 qinfo->scattered_rx = dev->data->scattered_rx; 654 qinfo->nb_desc = q->num_descs; 655 qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED; 656 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 657 } 658 659 static void __rte_cold 660 ionic_rx_empty(struct ionic_rx_qcq *rxq) 661 { 662 struct ionic_queue *q = &rxq->qcq.q; 663 struct rte_mbuf *mbuf; 664 void **info; 665 666 while (q->tail_idx != q->head_idx) { 667 info = IONIC_INFO_PTR(q, q->tail_idx); 668 mbuf = info[0]; 669 rte_mempool_put(rxq->mb_pool, mbuf); 670 671 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 672 } 673 } 674 675 void __rte_cold 676 ionic_dev_rx_queue_release(void *rx_queue) 677 { 678 struct ionic_rx_qcq *rxq = rx_queue; 679 struct ionic_rx_stats *stats; 680 681 if (!rxq) 682 return; 683 684 IONIC_PRINT_CALL(); 685 686 stats = &rxq->stats; 687 688 IONIC_PRINT(DEBUG, "RX queue %u pkts %ju mtod %ju", 689 rxq->qcq.q.index, stats->packets, stats->mtods); 690 691 ionic_rx_empty(rxq); 692 693 ionic_lif_rxq_deinit(rxq); 694 695 ionic_qcq_free(&rxq->qcq); 696 } 697 698 int __rte_cold 699 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, 700 uint16_t rx_queue_id, 701 uint16_t nb_desc, 702 uint32_t socket_id, 703 const struct rte_eth_rxconf *rx_conf, 704 struct rte_mempool *mp) 705 { 706 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 707 struct ionic_rx_qcq *rxq; 708 uint64_t offloads; 709 int err; 710 711 if (rx_queue_id >= lif->nrxqcqs) { 712 IONIC_PRINT(ERR, 713 "Queue index %u not available (max %u queues)", 714 rx_queue_id, lif->nrxqcqs); 715 return -EINVAL; 716 } 717 718 offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads; 719 IONIC_PRINT(DEBUG, 720 "Configuring skt %u RX queue %u with %u buffers, offloads %jx", 721 socket_id, rx_queue_id, nb_desc, offloads); 722 723 if (!rx_conf->rx_drop_en) 724 IONIC_PRINT(WARNING, "No-drop mode is not supported"); 725 726 /* Validate number of receive descriptors */ 727 if (!rte_is_power_of_2(nb_desc) || 728 nb_desc < IONIC_MIN_RING_DESC || 729 nb_desc > IONIC_MAX_RING_DESC) { 730 IONIC_PRINT(ERR, 731 "Bad descriptor count (%u) for queue %u (min: %u)", 732 nb_desc, rx_queue_id, IONIC_MIN_RING_DESC); 733 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 734 } 735 736 /* Free memory prior to re-allocation if needed... */ 737 if (eth_dev->data->rx_queues[rx_queue_id] != NULL) { 738 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id]; 739 ionic_dev_rx_queue_release(rx_queue); 740 eth_dev->data->rx_queues[rx_queue_id] = NULL; 741 } 742 743 eth_dev->data->rx_queue_state[rx_queue_id] = 744 RTE_ETH_QUEUE_STATE_STOPPED; 745 746 err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc, 747 &rxq); 748 if (err) { 749 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); 750 return -EINVAL; 751 } 752 753 rxq->mb_pool = mp; 754 755 /* 756 * Note: the interface does not currently support 757 * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN 758 * when the adapter will be able to keep the CRC and subtract 759 * it to the length for all received packets: 760 * if (eth_dev->data->dev_conf.rxmode.offloads & 761 * DEV_RX_OFFLOAD_KEEP_CRC) 762 * rxq->crc_len = ETHER_CRC_LEN; 763 */ 764 765 /* Do not start queue with rte_eth_dev_start() */ 766 if (rx_conf->rx_deferred_start) 767 rxq->flags |= IONIC_QCQ_F_DEFERRED; 768 769 eth_dev->data->rx_queues[rx_queue_id] = rxq; 770 771 return 0; 772 } 773 774 static __rte_always_inline void 775 ionic_rx_clean(struct ionic_rx_qcq *rxq, 776 uint32_t q_desc_index, uint32_t cq_desc_index, 777 void *service_cb_arg) 778 { 779 struct ionic_queue *q = &rxq->qcq.q; 780 struct ionic_cq *cq = &rxq->qcq.cq; 781 struct ionic_rxq_comp *cq_desc_base = cq->base; 782 struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 783 struct rte_mbuf *rxm, *rxm_seg; 784 uint32_t max_frame_size = 785 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 786 uint64_t pkt_flags = 0; 787 uint32_t pkt_type; 788 struct ionic_rx_stats *stats = &rxq->stats; 789 struct ionic_rx_service *recv_args = (struct ionic_rx_service *) 790 service_cb_arg; 791 uint32_t buf_size = (uint16_t) 792 (rte_pktmbuf_data_room_size(rxq->mb_pool) - 793 RTE_PKTMBUF_HEADROOM); 794 uint32_t left; 795 void **info; 796 797 assert(q_desc_index == cq_desc->comp_index); 798 799 info = IONIC_INFO_PTR(q, cq_desc->comp_index); 800 801 rxm = info[0]; 802 803 if (!recv_args) { 804 stats->no_cb_arg++; 805 /* Flush */ 806 rte_pktmbuf_free(rxm); 807 /* 808 * Note: rte_mempool_put is faster with no segs 809 * rte_mempool_put(rxq->mb_pool, rxm); 810 */ 811 return; 812 } 813 814 if (cq_desc->status) { 815 stats->bad_cq_status++; 816 ionic_rx_recycle(q, q_desc_index, rxm); 817 return; 818 } 819 820 if (recv_args->nb_rx >= recv_args->nb_pkts) { 821 stats->no_room++; 822 ionic_rx_recycle(q, q_desc_index, rxm); 823 return; 824 } 825 826 if (cq_desc->len > max_frame_size || 827 cq_desc->len == 0) { 828 stats->bad_len++; 829 ionic_rx_recycle(q, q_desc_index, rxm); 830 return; 831 } 832 833 rxm->data_off = RTE_PKTMBUF_HEADROOM; 834 rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); 835 rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ 836 rxm->pkt_len = cq_desc->len; 837 rxm->port = rxq->qcq.lif->port_id; 838 839 left = cq_desc->len; 840 841 rxm->data_len = RTE_MIN(buf_size, left); 842 left -= rxm->data_len; 843 844 rxm_seg = rxm->next; 845 while (rxm_seg && left) { 846 rxm_seg->data_len = RTE_MIN(buf_size, left); 847 left -= rxm_seg->data_len; 848 849 rxm_seg = rxm_seg->next; 850 rxm->nb_segs++; 851 } 852 853 /* RSS */ 854 pkt_flags |= PKT_RX_RSS_HASH; 855 rxm->hash.rss = cq_desc->rss_hash; 856 857 /* Vlan Strip */ 858 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 859 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 860 rxm->vlan_tci = cq_desc->vlan_tci; 861 } 862 863 /* Checksum */ 864 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 865 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK) 866 pkt_flags |= PKT_RX_IP_CKSUM_GOOD; 867 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD) 868 pkt_flags |= PKT_RX_IP_CKSUM_BAD; 869 870 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) || 871 (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK)) 872 pkt_flags |= PKT_RX_L4_CKSUM_GOOD; 873 else if ((cq_desc->csum_flags & 874 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 875 (cq_desc->csum_flags & 876 IONIC_RXQ_COMP_CSUM_F_UDP_BAD)) 877 pkt_flags |= PKT_RX_L4_CKSUM_BAD; 878 } 879 880 rxm->ol_flags = pkt_flags; 881 882 /* Packet Type */ 883 switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 884 case IONIC_PKT_TYPE_IPV4: 885 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 886 break; 887 case IONIC_PKT_TYPE_IPV6: 888 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 889 break; 890 case IONIC_PKT_TYPE_IPV4_TCP: 891 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 892 RTE_PTYPE_L4_TCP; 893 break; 894 case IONIC_PKT_TYPE_IPV6_TCP: 895 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 896 RTE_PTYPE_L4_TCP; 897 break; 898 case IONIC_PKT_TYPE_IPV4_UDP: 899 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 900 RTE_PTYPE_L4_UDP; 901 break; 902 case IONIC_PKT_TYPE_IPV6_UDP: 903 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 904 RTE_PTYPE_L4_UDP; 905 break; 906 default: 907 { 908 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 909 struct rte_ether_hdr *); 910 uint16_t ether_type = eth_h->ether_type; 911 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 912 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 913 else 914 pkt_type = RTE_PTYPE_UNKNOWN; 915 stats->mtods++; 916 break; 917 } 918 } 919 920 rxm->packet_type = pkt_type; 921 922 recv_args->rx_pkts[recv_args->nb_rx] = rxm; 923 recv_args->nb_rx++; 924 925 stats->packets++; 926 stats->bytes += rxm->pkt_len; 927 } 928 929 static void 930 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 931 struct rte_mbuf *mbuf) 932 { 933 struct ionic_rxq_desc *desc_base = q->base; 934 struct ionic_rxq_desc *old = &desc_base[q_desc_index]; 935 struct ionic_rxq_desc *new = &desc_base[q->head_idx]; 936 937 new->addr = old->addr; 938 new->len = old->len; 939 940 q->info[q->head_idx] = mbuf; 941 942 q->head_idx = Q_NEXT_TO_POST(q, 1); 943 944 ionic_q_flush(q); 945 } 946 947 static __rte_always_inline int 948 ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) 949 { 950 struct ionic_queue *q = &rxq->qcq.q; 951 struct ionic_rxq_desc *desc, *desc_base = q->base; 952 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; 953 struct ionic_rxq_sg_elem *elem; 954 void **info; 955 rte_iova_t dma_addr; 956 uint32_t i, j, nsegs, buf_size, size; 957 bool ring_doorbell; 958 959 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 960 RTE_PKTMBUF_HEADROOM); 961 962 /* Initialize software ring entries */ 963 for (i = ionic_q_space_avail(q); i; i--) { 964 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool); 965 struct rte_mbuf *prev_rxm_seg; 966 967 if (rxm == NULL) { 968 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 969 return -ENOMEM; 970 } 971 972 info = IONIC_INFO_PTR(q, q->head_idx); 973 974 nsegs = (len + buf_size - 1) / buf_size; 975 976 desc = &desc_base[q->head_idx]; 977 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm)); 978 desc->addr = dma_addr; 979 desc->len = buf_size; 980 size = buf_size; 981 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 982 IONIC_RXQ_DESC_OPCODE_SIMPLE; 983 rxm->next = NULL; 984 985 prev_rxm_seg = rxm; 986 sg_desc = &sg_desc_base[q->head_idx]; 987 elem = sg_desc->elems; 988 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) { 989 struct rte_mbuf *rxm_seg; 990 rte_iova_t data_iova; 991 992 rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool); 993 if (rxm_seg == NULL) { 994 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 995 return -ENOMEM; 996 } 997 998 data_iova = rte_mbuf_data_iova(rxm_seg); 999 dma_addr = rte_cpu_to_le_64(data_iova); 1000 elem->addr = dma_addr; 1001 elem->len = buf_size; 1002 size += buf_size; 1003 elem++; 1004 rxm_seg->next = NULL; 1005 prev_rxm_seg->next = rxm_seg; 1006 prev_rxm_seg = rxm_seg; 1007 } 1008 1009 if (size < len) 1010 IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)", 1011 size, len); 1012 1013 ring_doorbell = ((q->head_idx + 1) & 1014 IONIC_RX_RING_DOORBELL_STRIDE) == 0; 1015 1016 info[0] = rxm; 1017 1018 q->head_idx = Q_NEXT_TO_POST(q, 1); 1019 1020 if (ring_doorbell) 1021 ionic_q_flush(q); 1022 } 1023 1024 return 0; 1025 } 1026 1027 /* 1028 * Start Receive Units for specified queue. 1029 */ 1030 int __rte_cold 1031 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1032 { 1033 uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 1034 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; 1035 struct ionic_rx_qcq *rxq; 1036 int err; 1037 1038 if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 1039 IONIC_PRINT(DEBUG, "RX queue %u already started", 1040 rx_queue_id); 1041 return 0; 1042 } 1043 1044 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1045 1046 IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", 1047 rx_queue_id, rxq->qcq.q.num_descs, frame_size); 1048 1049 if (!(rxq->flags & IONIC_QCQ_F_INITED)) { 1050 err = ionic_lif_rxq_init(rxq); 1051 if (err) 1052 return err; 1053 } else { 1054 ionic_qcq_enable(&rxq->qcq); 1055 } 1056 1057 /* Allocate buffers for descriptor rings */ 1058 if (ionic_rx_fill(rxq, frame_size) != 0) { 1059 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d", 1060 rx_queue_id); 1061 return -1; 1062 } 1063 1064 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1065 1066 return 0; 1067 } 1068 1069 static __rte_always_inline void 1070 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 1071 void *service_cb_arg) 1072 { 1073 struct ionic_cq *cq = &rxq->qcq.cq; 1074 struct ionic_queue *q = &rxq->qcq.q; 1075 struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; 1076 bool more; 1077 uint32_t curr_q_tail_idx, curr_cq_tail_idx; 1078 uint32_t work_done = 0; 1079 1080 if (work_to_do == 0) 1081 return; 1082 1083 cq_desc = &cq_desc_base[cq->tail_idx]; 1084 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 1085 curr_cq_tail_idx = cq->tail_idx; 1086 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 1087 1088 if (cq->tail_idx == 0) 1089 cq->done_color = !cq->done_color; 1090 1091 /* Prefetch the next 4 descriptors */ 1092 if ((cq->tail_idx & 0x3) == 0) 1093 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 1094 1095 do { 1096 more = (q->tail_idx != cq_desc->comp_index); 1097 1098 curr_q_tail_idx = q->tail_idx; 1099 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 1100 1101 /* Prefetch the next 4 descriptors */ 1102 if ((q->tail_idx & 0x3) == 0) 1103 /* q desc info */ 1104 rte_prefetch0(&q->info[q->tail_idx]); 1105 1106 ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx, 1107 service_cb_arg); 1108 1109 } while (more); 1110 1111 if (++work_done == work_to_do) 1112 break; 1113 1114 cq_desc = &cq_desc_base[cq->tail_idx]; 1115 } 1116 } 1117 1118 /* 1119 * Stop Receive Units for specified queue. 1120 */ 1121 int __rte_cold 1122 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1123 { 1124 struct ionic_rx_qcq *rxq; 1125 1126 IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); 1127 1128 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1129 1130 eth_dev->data->rx_queue_state[rx_queue_id] = 1131 RTE_ETH_QUEUE_STATE_STOPPED; 1132 1133 ionic_qcq_disable(&rxq->qcq); 1134 1135 /* Flush */ 1136 ionic_rxq_service(rxq, -1, NULL); 1137 1138 return 0; 1139 } 1140 1141 uint16_t 1142 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1143 uint16_t nb_pkts) 1144 { 1145 struct ionic_rx_qcq *rxq = rx_queue; 1146 uint32_t frame_size = 1147 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 1148 struct ionic_rx_service service_cb_arg; 1149 1150 service_cb_arg.rx_pkts = rx_pkts; 1151 service_cb_arg.nb_pkts = nb_pkts; 1152 service_cb_arg.nb_rx = 0; 1153 1154 ionic_rxq_service(rxq, nb_pkts, &service_cb_arg); 1155 1156 ionic_rx_fill(rxq, frame_size); 1157 1158 return service_cb_arg.nb_rx; 1159 } 1160