1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2022 Advanced Micro Devices, Inc. 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <stdint.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 #include <rte_log.h> 19 #include <rte_debug.h> 20 #include <rte_interrupts.h> 21 #include <rte_pci.h> 22 #include <rte_memory.h> 23 #include <rte_memzone.h> 24 #include <rte_launch.h> 25 #include <rte_eal.h> 26 #include <rte_per_lcore.h> 27 #include <rte_lcore.h> 28 #include <rte_atomic.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_mempool.h> 31 #include <rte_malloc.h> 32 #include <rte_mbuf.h> 33 #include <rte_ether.h> 34 #include <ethdev_driver.h> 35 #include <rte_prefetch.h> 36 #include <rte_udp.h> 37 #include <rte_tcp.h> 38 #include <rte_sctp.h> 39 #include <rte_string_fns.h> 40 #include <rte_errno.h> 41 #include <rte_ip.h> 42 #include <rte_net.h> 43 44 #include "ionic_logs.h" 45 #include "ionic_mac_api.h" 46 #include "ionic_ethdev.h" 47 #include "ionic_lif.h" 48 #include "ionic_rxtx.h" 49 50 static void 51 ionic_empty_array(void **array, uint32_t cnt, uint16_t idx) 52 { 53 uint32_t i; 54 55 for (i = idx; i < cnt; i++) 56 if (array[i]) 57 rte_pktmbuf_free_seg(array[i]); 58 59 memset(array, 0, sizeof(void *) * cnt); 60 } 61 62 static void __rte_cold 63 ionic_tx_empty(struct ionic_tx_qcq *txq) 64 { 65 struct ionic_queue *q = &txq->qcq.q; 66 67 ionic_empty_array(q->info, q->num_descs, 0); 68 } 69 70 static void __rte_cold 71 ionic_rx_empty(struct ionic_rx_qcq *rxq) 72 { 73 struct ionic_queue *q = &rxq->qcq.q; 74 75 /* 76 * Walk the full info array so that the clean up includes any 77 * fragments that were left dangling for later reuse 78 */ 79 ionic_empty_array(q->info, q->num_descs * q->num_segs, 0); 80 } 81 82 /********************************************************************* 83 * 84 * TX functions 85 * 86 **********************************************************************/ 87 88 void 89 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 90 struct rte_eth_txq_info *qinfo) 91 { 92 struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id]; 93 struct ionic_queue *q = &txq->qcq.q; 94 95 qinfo->nb_desc = q->num_descs; 96 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 97 qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED; 98 } 99 100 static __rte_always_inline void 101 ionic_tx_flush(struct ionic_tx_qcq *txq) 102 { 103 struct ionic_cq *cq = &txq->qcq.cq; 104 struct ionic_queue *q = &txq->qcq.q; 105 struct rte_mbuf *txm, *next; 106 struct ionic_txq_comp *cq_desc_base = cq->base; 107 struct ionic_txq_comp *cq_desc; 108 void **info; 109 u_int32_t comp_index = (u_int32_t)-1; 110 111 cq_desc = &cq_desc_base[cq->tail_idx]; 112 while (color_match(cq_desc->color, cq->done_color)) { 113 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 114 115 /* Prefetch the next 4 descriptors (not really useful here) */ 116 if ((cq->tail_idx & 0x3) == 0) 117 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 118 119 if (cq->tail_idx == 0) 120 cq->done_color = !cq->done_color; 121 122 comp_index = cq_desc->comp_index; 123 124 cq_desc = &cq_desc_base[cq->tail_idx]; 125 } 126 127 if (comp_index != (u_int32_t)-1) { 128 while (q->tail_idx != comp_index) { 129 info = IONIC_INFO_PTR(q, q->tail_idx); 130 131 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 132 133 /* Prefetch the next 4 descriptors */ 134 if ((q->tail_idx & 0x3) == 0) 135 /* q desc info */ 136 rte_prefetch0(&q->info[q->tail_idx]); 137 138 /* 139 * Note: you can just use rte_pktmbuf_free, 140 * but this loop is faster 141 */ 142 txm = info[0]; 143 while (txm != NULL) { 144 next = txm->next; 145 rte_pktmbuf_free_seg(txm); 146 txm = next; 147 } 148 } 149 } 150 } 151 152 void __rte_cold 153 ionic_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 154 { 155 struct ionic_tx_qcq *txq = dev->data->tx_queues[qid]; 156 157 IONIC_PRINT_CALL(); 158 159 ionic_qcq_free(&txq->qcq); 160 } 161 162 int __rte_cold 163 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 164 { 165 struct ionic_tx_stats *stats; 166 struct ionic_tx_qcq *txq; 167 168 IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); 169 170 txq = eth_dev->data->tx_queues[tx_queue_id]; 171 172 eth_dev->data->tx_queue_state[tx_queue_id] = 173 RTE_ETH_QUEUE_STATE_STOPPED; 174 175 /* 176 * Note: we should better post NOP Tx desc and wait for its completion 177 * before disabling Tx queue 178 */ 179 180 ionic_lif_txq_deinit(txq); 181 182 /* Free all buffers from descriptor ring */ 183 ionic_tx_empty(txq); 184 185 stats = &txq->stats; 186 IONIC_PRINT(DEBUG, "TX queue %u pkts %ju tso %ju", 187 txq->qcq.q.index, stats->packets, stats->tso); 188 189 return 0; 190 } 191 192 int __rte_cold 193 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, 194 uint16_t nb_desc, uint32_t socket_id, 195 const struct rte_eth_txconf *tx_conf) 196 { 197 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 198 struct ionic_tx_qcq *txq; 199 uint64_t offloads; 200 int err; 201 202 if (tx_queue_id >= lif->ntxqcqs) { 203 IONIC_PRINT(DEBUG, "Queue index %u not available " 204 "(max %u queues)", 205 tx_queue_id, lif->ntxqcqs); 206 return -EINVAL; 207 } 208 209 offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads; 210 IONIC_PRINT(DEBUG, 211 "Configuring skt %u TX queue %u with %u buffers, offloads %jx", 212 socket_id, tx_queue_id, nb_desc, offloads); 213 214 /* Validate number of receive descriptors */ 215 if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC) 216 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 217 218 /* Free memory prior to re-allocation if needed... */ 219 if (eth_dev->data->tx_queues[tx_queue_id] != NULL) { 220 ionic_dev_tx_queue_release(eth_dev, tx_queue_id); 221 eth_dev->data->tx_queues[tx_queue_id] = NULL; 222 } 223 224 eth_dev->data->tx_queue_state[tx_queue_id] = 225 RTE_ETH_QUEUE_STATE_STOPPED; 226 227 err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq); 228 if (err) { 229 IONIC_PRINT(DEBUG, "Queue allocation failure"); 230 return -EINVAL; 231 } 232 233 /* Do not start queue with rte_eth_dev_start() */ 234 if (tx_conf->tx_deferred_start) 235 txq->flags |= IONIC_QCQ_F_DEFERRED; 236 237 /* Convert the offload flags into queue flags */ 238 if (offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) 239 txq->flags |= IONIC_QCQ_F_CSUM_L3; 240 if (offloads & RTE_ETH_TX_OFFLOAD_TCP_CKSUM) 241 txq->flags |= IONIC_QCQ_F_CSUM_TCP; 242 if (offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) 243 txq->flags |= IONIC_QCQ_F_CSUM_UDP; 244 245 eth_dev->data->tx_queues[tx_queue_id] = txq; 246 247 return 0; 248 } 249 250 /* 251 * Start Transmit Units for specified queue. 252 */ 253 int __rte_cold 254 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 255 { 256 uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; 257 struct ionic_tx_qcq *txq; 258 int err; 259 260 if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 261 IONIC_PRINT(DEBUG, "TX queue %u already started", 262 tx_queue_id); 263 return 0; 264 } 265 266 txq = eth_dev->data->tx_queues[tx_queue_id]; 267 268 IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", 269 tx_queue_id, txq->qcq.q.num_descs); 270 271 err = ionic_lif_txq_init(txq); 272 if (err) 273 return err; 274 275 tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 276 277 return 0; 278 } 279 280 static void 281 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm) 282 { 283 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 284 char *l3_hdr = ((char *)eth_hdr) + txm->l2_len; 285 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 286 (l3_hdr + txm->l3_len); 287 288 if (txm->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 289 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 290 ipv4_hdr->hdr_checksum = 0; 291 tcp_hdr->cksum = 0; 292 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 293 } else { 294 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 295 tcp_hdr->cksum = 0; 296 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 297 } 298 } 299 300 static void 301 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm) 302 { 303 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 304 char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len + 305 txm->outer_l3_len + txm->l2_len; 306 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 307 (l3_hdr + txm->l3_len); 308 309 if (txm->ol_flags & RTE_MBUF_F_TX_IPV4) { 310 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 311 ipv4_hdr->hdr_checksum = 0; 312 tcp_hdr->cksum = 0; 313 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 314 } else { 315 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 316 tcp_hdr->cksum = 0; 317 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 318 } 319 } 320 321 static void 322 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 323 struct rte_mbuf *txm, 324 rte_iova_t addr, uint8_t nsge, uint16_t len, 325 uint32_t hdrlen, uint32_t mss, 326 bool encap, 327 uint16_t vlan_tci, bool has_vlan, 328 bool start, bool done) 329 { 330 void **info; 331 uint64_t cmd; 332 uint8_t flags = 0; 333 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 334 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 335 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 336 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 337 338 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, 339 flags, nsge, addr); 340 desc->cmd = rte_cpu_to_le_64(cmd); 341 desc->len = rte_cpu_to_le_16(len); 342 desc->vlan_tci = rte_cpu_to_le_16(vlan_tci); 343 desc->hdr_len = rte_cpu_to_le_16(hdrlen); 344 desc->mss = rte_cpu_to_le_16(mss); 345 346 if (done) { 347 info = IONIC_INFO_PTR(q, q->head_idx); 348 info[0] = txm; 349 } 350 351 q->head_idx = Q_NEXT_TO_POST(q, 1); 352 } 353 354 static struct ionic_txq_desc * 355 ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) 356 { 357 struct ionic_queue *q = &txq->qcq.q; 358 struct ionic_txq_desc *desc_base = q->base; 359 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 360 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 361 struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; 362 363 *elem = sg_desc->elems; 364 return desc; 365 } 366 367 static int 368 ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) 369 { 370 struct ionic_queue *q = &txq->qcq.q; 371 struct ionic_tx_stats *stats = &txq->stats; 372 struct ionic_txq_desc *desc; 373 struct ionic_txq_sg_elem *elem; 374 struct rte_mbuf *txm_seg; 375 rte_iova_t data_iova; 376 uint64_t desc_addr = 0, next_addr; 377 uint16_t desc_len = 0; 378 uint8_t desc_nsge; 379 uint32_t hdrlen; 380 uint32_t mss = txm->tso_segsz; 381 uint32_t frag_left = 0; 382 uint32_t left; 383 uint32_t seglen; 384 uint32_t len; 385 uint32_t offset = 0; 386 bool start, done; 387 bool encap; 388 bool has_vlan = !!(txm->ol_flags & RTE_MBUF_F_TX_VLAN); 389 uint16_t vlan_tci = txm->vlan_tci; 390 uint64_t ol_flags = txm->ol_flags; 391 392 encap = ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || 393 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && 394 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || 395 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)); 396 397 /* Preload inner-most TCP csum field with IP pseudo hdr 398 * calculated with IP length set to zero. HW will later 399 * add in length to each TCP segment resulting from the TSO. 400 */ 401 402 if (encap) { 403 ionic_tx_tcp_inner_pseudo_csum(txm); 404 hdrlen = txm->outer_l2_len + txm->outer_l3_len + 405 txm->l2_len + txm->l3_len + txm->l4_len; 406 } else { 407 ionic_tx_tcp_pseudo_csum(txm); 408 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len; 409 } 410 411 seglen = hdrlen + mss; 412 left = txm->data_len; 413 data_iova = rte_mbuf_data_iova(txm); 414 415 desc = ionic_tx_tso_next(txq, &elem); 416 start = true; 417 418 /* Chop data up into desc segments */ 419 420 while (left > 0) { 421 len = RTE_MIN(seglen, left); 422 frag_left = seglen - len; 423 desc_addr = rte_cpu_to_le_64(data_iova + offset); 424 desc_len = len; 425 desc_nsge = 0; 426 left -= len; 427 offset += len; 428 if (txm->nb_segs > 1 && frag_left > 0) 429 continue; 430 done = (txm->nb_segs == 1 && left == 0); 431 ionic_tx_tso_post(q, desc, txm, 432 desc_addr, desc_nsge, desc_len, 433 hdrlen, mss, 434 encap, 435 vlan_tci, has_vlan, 436 start, done); 437 desc = ionic_tx_tso_next(txq, &elem); 438 start = false; 439 seglen = mss; 440 } 441 442 /* Chop frags into desc segments */ 443 444 txm_seg = txm->next; 445 while (txm_seg != NULL) { 446 offset = 0; 447 data_iova = rte_mbuf_data_iova(txm_seg); 448 left = txm_seg->data_len; 449 450 while (left > 0) { 451 next_addr = rte_cpu_to_le_64(data_iova + offset); 452 if (frag_left > 0) { 453 len = RTE_MIN(frag_left, left); 454 frag_left -= len; 455 elem->addr = next_addr; 456 elem->len = rte_cpu_to_le_16(len); 457 elem++; 458 desc_nsge++; 459 } else { 460 len = RTE_MIN(mss, left); 461 frag_left = mss - len; 462 desc_addr = next_addr; 463 desc_len = len; 464 desc_nsge = 0; 465 } 466 left -= len; 467 offset += len; 468 if (txm_seg->next != NULL && frag_left > 0) 469 continue; 470 471 done = (txm_seg->next == NULL && left == 0); 472 ionic_tx_tso_post(q, desc, txm_seg, 473 desc_addr, desc_nsge, desc_len, 474 hdrlen, mss, 475 encap, 476 vlan_tci, has_vlan, 477 start, done); 478 desc = ionic_tx_tso_next(txq, &elem); 479 start = false; 480 } 481 482 txm_seg = txm_seg->next; 483 } 484 485 stats->tso++; 486 487 return 0; 488 } 489 490 static __rte_always_inline int 491 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) 492 { 493 struct ionic_queue *q = &txq->qcq.q; 494 struct ionic_txq_desc *desc, *desc_base = q->base; 495 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 496 struct ionic_txq_sg_elem *elem; 497 struct ionic_tx_stats *stats = &txq->stats; 498 struct rte_mbuf *txm_seg; 499 void **info; 500 bool encap; 501 bool has_vlan; 502 uint64_t ol_flags = txm->ol_flags; 503 uint64_t addr, cmd; 504 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 505 uint8_t flags = 0; 506 507 desc = &desc_base[q->head_idx]; 508 info = IONIC_INFO_PTR(q, q->head_idx); 509 510 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && 511 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 512 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 513 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 514 } 515 516 if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && 517 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 518 ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && 519 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 520 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 521 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 522 } 523 524 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 525 stats->no_csum++; 526 527 has_vlan = (ol_flags & RTE_MBUF_F_TX_VLAN); 528 encap = ((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || 529 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && 530 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || 531 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6)); 532 533 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 534 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 535 536 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 537 538 cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 539 desc->cmd = rte_cpu_to_le_64(cmd); 540 desc->len = rte_cpu_to_le_16(txm->data_len); 541 desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); 542 543 info[0] = txm; 544 545 elem = sg_desc_base[q->head_idx].elems; 546 547 txm_seg = txm->next; 548 while (txm_seg != NULL) { 549 elem->len = rte_cpu_to_le_16(txm_seg->data_len); 550 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); 551 elem++; 552 txm_seg = txm_seg->next; 553 } 554 555 q->head_idx = Q_NEXT_TO_POST(q, 1); 556 557 return 0; 558 } 559 560 uint16_t 561 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 562 uint16_t nb_pkts) 563 { 564 struct ionic_tx_qcq *txq = tx_queue; 565 struct ionic_queue *q = &txq->qcq.q; 566 struct ionic_tx_stats *stats = &txq->stats; 567 struct rte_mbuf *mbuf; 568 uint32_t next_q_head_idx; 569 uint32_t bytes_tx = 0; 570 uint16_t nb_avail, nb_tx = 0; 571 int err; 572 573 /* Cleaning old buffers */ 574 ionic_tx_flush(txq); 575 576 nb_avail = ionic_q_space_avail(q); 577 if (unlikely(nb_avail < nb_pkts)) { 578 stats->stop += nb_pkts - nb_avail; 579 nb_pkts = nb_avail; 580 } 581 582 while (nb_tx < nb_pkts) { 583 next_q_head_idx = Q_NEXT_TO_POST(q, 1); 584 if ((next_q_head_idx & 0x3) == 0) { 585 struct ionic_txq_desc *desc_base = q->base; 586 rte_prefetch0(&desc_base[next_q_head_idx]); 587 rte_prefetch0(&q->info[next_q_head_idx]); 588 } 589 590 mbuf = tx_pkts[nb_tx]; 591 592 if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 593 err = ionic_tx_tso(txq, mbuf); 594 else 595 err = ionic_tx(txq, mbuf); 596 if (err) { 597 stats->drop += nb_pkts - nb_tx; 598 break; 599 } 600 601 bytes_tx += mbuf->pkt_len; 602 nb_tx++; 603 } 604 605 if (nb_tx > 0) { 606 rte_wmb(); 607 ionic_q_flush(q); 608 } 609 610 stats->packets += nb_tx; 611 stats->bytes += bytes_tx; 612 613 return nb_tx; 614 } 615 616 /********************************************************************* 617 * 618 * TX prep functions 619 * 620 **********************************************************************/ 621 622 #define IONIC_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_IPV4 | \ 623 RTE_MBUF_F_TX_IPV6 | \ 624 RTE_MBUF_F_TX_VLAN | \ 625 RTE_MBUF_F_TX_IP_CKSUM | \ 626 RTE_MBUF_F_TX_TCP_SEG | \ 627 RTE_MBUF_F_TX_L4_MASK) 628 629 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \ 630 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) 631 632 uint16_t 633 ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 634 { 635 struct ionic_tx_qcq *txq = tx_queue; 636 struct rte_mbuf *txm; 637 uint64_t offloads; 638 int i = 0; 639 640 for (i = 0; i < nb_pkts; i++) { 641 txm = tx_pkts[i]; 642 643 if (txm->nb_segs > txq->num_segs_fw) { 644 rte_errno = -EINVAL; 645 break; 646 } 647 648 offloads = txm->ol_flags; 649 650 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) { 651 rte_errno = -ENOTSUP; 652 break; 653 } 654 } 655 656 return i; 657 } 658 659 /********************************************************************* 660 * 661 * RX functions 662 * 663 **********************************************************************/ 664 665 void 666 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 667 struct rte_eth_rxq_info *qinfo) 668 { 669 struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id]; 670 struct ionic_queue *q = &rxq->qcq.q; 671 672 qinfo->mp = rxq->mb_pool; 673 qinfo->scattered_rx = dev->data->scattered_rx; 674 qinfo->nb_desc = q->num_descs; 675 qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED; 676 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 677 } 678 679 void __rte_cold 680 ionic_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 681 { 682 struct ionic_rx_qcq *rxq = dev->data->rx_queues[qid]; 683 684 if (!rxq) 685 return; 686 687 IONIC_PRINT_CALL(); 688 689 ionic_qcq_free(&rxq->qcq); 690 } 691 692 int __rte_cold 693 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, 694 uint16_t rx_queue_id, 695 uint16_t nb_desc, 696 uint32_t socket_id, 697 const struct rte_eth_rxconf *rx_conf, 698 struct rte_mempool *mp) 699 { 700 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 701 struct ionic_rx_qcq *rxq; 702 uint64_t offloads; 703 int err; 704 705 if (rx_queue_id >= lif->nrxqcqs) { 706 IONIC_PRINT(ERR, 707 "Queue index %u not available (max %u queues)", 708 rx_queue_id, lif->nrxqcqs); 709 return -EINVAL; 710 } 711 712 offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads; 713 IONIC_PRINT(DEBUG, 714 "Configuring skt %u RX queue %u with %u buffers, offloads %jx", 715 socket_id, rx_queue_id, nb_desc, offloads); 716 717 if (!rx_conf->rx_drop_en) 718 IONIC_PRINT(WARNING, "No-drop mode is not supported"); 719 720 /* Validate number of receive descriptors */ 721 if (!rte_is_power_of_2(nb_desc) || 722 nb_desc < IONIC_MIN_RING_DESC || 723 nb_desc > IONIC_MAX_RING_DESC) { 724 IONIC_PRINT(ERR, 725 "Bad descriptor count (%u) for queue %u (min: %u)", 726 nb_desc, rx_queue_id, IONIC_MIN_RING_DESC); 727 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 728 } 729 730 /* Free memory prior to re-allocation if needed... */ 731 if (eth_dev->data->rx_queues[rx_queue_id] != NULL) { 732 ionic_dev_rx_queue_release(eth_dev, rx_queue_id); 733 eth_dev->data->rx_queues[rx_queue_id] = NULL; 734 } 735 736 eth_dev->data->rx_queue_state[rx_queue_id] = 737 RTE_ETH_QUEUE_STATE_STOPPED; 738 739 err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc, mp, 740 &rxq); 741 if (err) { 742 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); 743 return -EINVAL; 744 } 745 746 rxq->mb_pool = mp; 747 748 /* 749 * Note: the interface does not currently support 750 * RTE_ETH_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN 751 * when the adapter will be able to keep the CRC and subtract 752 * it to the length for all received packets: 753 * if (eth_dev->data->dev_conf.rxmode.offloads & 754 * RTE_ETH_RX_OFFLOAD_KEEP_CRC) 755 * rxq->crc_len = ETHER_CRC_LEN; 756 */ 757 758 /* Do not start queue with rte_eth_dev_start() */ 759 if (rx_conf->rx_deferred_start) 760 rxq->flags |= IONIC_QCQ_F_DEFERRED; 761 762 eth_dev->data->rx_queues[rx_queue_id] = rxq; 763 764 return 0; 765 } 766 767 /* 768 * Cleans one descriptor. Connects the filled mbufs into a chain. 769 * Does not advance the tail index. 770 */ 771 static __rte_always_inline void 772 ionic_rx_clean_one(struct ionic_rx_qcq *rxq, 773 struct ionic_rxq_comp *cq_desc, 774 struct ionic_rx_service *rx_svc) 775 { 776 struct ionic_queue *q = &rxq->qcq.q; 777 struct rte_mbuf *rxm, *rxm_seg, *prev_rxm; 778 struct ionic_rx_stats *stats = &rxq->stats; 779 uint64_t pkt_flags = 0; 780 uint32_t pkt_type; 781 uint32_t left, i; 782 uint16_t cq_desc_len; 783 void **info; 784 785 cq_desc_len = rte_le_to_cpu_16(cq_desc->len); 786 787 info = IONIC_INFO_PTR(q, q->tail_idx); 788 789 rxm = info[0]; 790 791 if (cq_desc->status) { 792 stats->bad_cq_status++; 793 return; 794 } 795 796 if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { 797 stats->bad_len++; 798 return; 799 } 800 801 info[0] = NULL; 802 803 /* Set the mbuf metadata based on the cq entry */ 804 rxm->rearm_data[0] = rxq->rearm_data; 805 rxm->pkt_len = cq_desc_len; 806 rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len); 807 left = cq_desc_len - rxm->data_len; 808 rxm->nb_segs = cq_desc->num_sg_elems + 1; 809 prev_rxm = rxm; 810 811 for (i = 1; i < rxm->nb_segs && left; i++) { 812 rxm_seg = info[i]; 813 info[i] = NULL; 814 815 /* Set the chained mbuf metadata */ 816 rxm_seg->rearm_data[0] = rxq->rearm_seg_data; 817 rxm_seg->data_len = RTE_MIN(rxq->seg_size, left); 818 left -= rxm_seg->data_len; 819 820 /* Link the mbuf */ 821 prev_rxm->next = rxm_seg; 822 prev_rxm = rxm_seg; 823 } 824 825 /* Terminate the mbuf chain */ 826 prev_rxm->next = NULL; 827 828 /* RSS */ 829 pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; 830 rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); 831 832 /* Vlan Strip */ 833 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 834 pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 835 rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); 836 } 837 838 /* Checksum */ 839 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 840 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK) 841 pkt_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 842 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD) 843 pkt_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 844 845 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) || 846 (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK)) 847 pkt_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 848 else if ((cq_desc->csum_flags & 849 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 850 (cq_desc->csum_flags & 851 IONIC_RXQ_COMP_CSUM_F_UDP_BAD)) 852 pkt_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 853 } 854 855 rxm->ol_flags = pkt_flags; 856 857 /* Packet Type */ 858 switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 859 case IONIC_PKT_TYPE_IPV4: 860 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 861 break; 862 case IONIC_PKT_TYPE_IPV6: 863 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 864 break; 865 case IONIC_PKT_TYPE_IPV4_TCP: 866 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 867 RTE_PTYPE_L4_TCP; 868 break; 869 case IONIC_PKT_TYPE_IPV6_TCP: 870 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 871 RTE_PTYPE_L4_TCP; 872 break; 873 case IONIC_PKT_TYPE_IPV4_UDP: 874 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 875 RTE_PTYPE_L4_UDP; 876 break; 877 case IONIC_PKT_TYPE_IPV6_UDP: 878 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 879 RTE_PTYPE_L4_UDP; 880 break; 881 default: 882 { 883 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 884 struct rte_ether_hdr *); 885 uint16_t ether_type = eth_h->ether_type; 886 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 887 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 888 else 889 pkt_type = RTE_PTYPE_UNKNOWN; 890 stats->mtods++; 891 break; 892 } 893 } 894 895 rxm->packet_type = pkt_type; 896 897 rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; 898 rx_svc->nb_rx++; 899 900 stats->packets++; 901 stats->bytes += rxm->pkt_len; 902 } 903 904 /* 905 * Fills one descriptor with mbufs. Does not advance the head index. 906 */ 907 static __rte_always_inline int 908 ionic_rx_fill_one(struct ionic_rx_qcq *rxq) 909 { 910 struct ionic_queue *q = &rxq->qcq.q; 911 struct rte_mbuf *rxm, *rxm_seg; 912 struct ionic_rxq_desc *desc, *desc_base = q->base; 913 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; 914 rte_iova_t data_iova; 915 uint32_t i; 916 void **info; 917 918 info = IONIC_INFO_PTR(q, q->head_idx); 919 desc = &desc_base[q->head_idx]; 920 sg_desc = &sg_desc_base[q->head_idx]; 921 922 /* mbuf is unused => whole chain is unused */ 923 if (unlikely(info[0])) 924 return 0; 925 926 rxm = rte_mbuf_raw_alloc(rxq->mb_pool); 927 if (unlikely(rxm == NULL)) { 928 assert(0); 929 return -ENOMEM; 930 } 931 932 info[0] = rxm; 933 934 data_iova = rte_mbuf_data_iova_default(rxm); 935 desc->addr = rte_cpu_to_le_64(data_iova); 936 937 for (i = 1; i < q->num_segs; i++) { 938 /* mbuf is unused => rest of the chain is unused */ 939 if (info[i]) 940 return 0; 941 942 rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool); 943 if (rxm_seg == NULL) { 944 assert(0); 945 return -ENOMEM; 946 } 947 948 info[i] = rxm_seg; 949 950 /* The data_off does not get set to 0 until later */ 951 data_iova = rxm_seg->buf_iova; 952 sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova); 953 } 954 955 return 0; 956 } 957 958 /* 959 * Fills all descriptors with mbufs. 960 */ 961 static int __rte_cold 962 ionic_rx_fill(struct ionic_rx_qcq *rxq) 963 { 964 struct ionic_queue *q = &rxq->qcq.q; 965 uint32_t i; 966 int err; 967 968 for (i = 1; i < q->num_descs; i++) { 969 err = ionic_rx_fill_one(rxq); 970 if (err) 971 return err; 972 973 q->head_idx = Q_NEXT_TO_POST(q, 1); 974 } 975 976 ionic_q_flush(q); 977 978 return 0; 979 } 980 981 /* 982 * Perform one-time initialization of descriptor fields 983 * which will not change for the life of the queue. 984 */ 985 static void __rte_cold 986 ionic_rx_init_descriptors(struct ionic_rx_qcq *rxq) 987 { 988 struct ionic_queue *q = &rxq->qcq.q; 989 struct ionic_rxq_desc *desc, *desc_base = q->base; 990 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; 991 uint32_t i, j; 992 uint8_t opcode; 993 994 opcode = (q->num_segs > 1) ? 995 IONIC_RXQ_DESC_OPCODE_SG : IONIC_RXQ_DESC_OPCODE_SIMPLE; 996 997 /* 998 * NB: Only the first segment needs to leave headroom (hdr_seg_size). 999 * Later segments (seg_size) do not. 1000 */ 1001 for (i = 0; i < q->num_descs; i++) { 1002 desc = &desc_base[i]; 1003 desc->len = rte_cpu_to_le_16(rxq->hdr_seg_size); 1004 desc->opcode = opcode; 1005 1006 sg_desc = &sg_desc_base[i]; 1007 for (j = 0; j < q->num_segs - 1u; j++) 1008 sg_desc->elems[j].len = 1009 rte_cpu_to_le_16(rxq->seg_size); 1010 } 1011 } 1012 1013 /* 1014 * Start Receive Units for specified queue. 1015 */ 1016 int __rte_cold 1017 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1018 { 1019 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; 1020 struct ionic_rx_qcq *rxq; 1021 struct ionic_queue *q; 1022 int err; 1023 1024 if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 1025 IONIC_PRINT(DEBUG, "RX queue %u already started", 1026 rx_queue_id); 1027 return 0; 1028 } 1029 1030 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1031 q = &rxq->qcq.q; 1032 1033 rxq->frame_size = rxq->qcq.lif->frame_size - RTE_ETHER_CRC_LEN; 1034 1035 /* Recalculate segment count based on MTU */ 1036 q->num_segs = 1 + 1037 (rxq->frame_size + RTE_PKTMBUF_HEADROOM - 1) / rxq->seg_size; 1038 1039 IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs, size %u segs %u", 1040 rx_queue_id, q->num_descs, rxq->frame_size, q->num_segs); 1041 1042 ionic_rx_init_descriptors(rxq); 1043 1044 err = ionic_lif_rxq_init(rxq); 1045 if (err) 1046 return err; 1047 1048 /* Allocate buffers for descriptor rings */ 1049 if (ionic_rx_fill(rxq) != 0) { 1050 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d", 1051 rx_queue_id); 1052 return -1; 1053 } 1054 1055 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1056 1057 return 0; 1058 } 1059 1060 /* 1061 * Walk the CQ to find completed receive descriptors. 1062 * Any completed descriptor found is refilled. 1063 */ 1064 static __rte_always_inline void 1065 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 1066 struct ionic_rx_service *rx_svc) 1067 { 1068 struct ionic_cq *cq = &rxq->qcq.cq; 1069 struct ionic_queue *q = &rxq->qcq.q; 1070 struct ionic_rxq_desc *q_desc_base = q->base; 1071 struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; 1072 uint32_t work_done = 0; 1073 1074 cq_desc = &cq_desc_base[cq->tail_idx]; 1075 1076 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 1077 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 1078 1079 if (cq->tail_idx == 0) 1080 cq->done_color = !cq->done_color; 1081 1082 /* Prefetch 8 x 8B bufinfo */ 1083 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8))); 1084 /* Prefetch 4 x 16B comp */ 1085 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 1086 /* Prefetch 4 x 16B descriptors */ 1087 rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); 1088 1089 ionic_rx_clean_one(rxq, cq_desc, rx_svc); 1090 1091 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 1092 1093 (void)ionic_rx_fill_one(rxq); 1094 1095 q->head_idx = Q_NEXT_TO_POST(q, 1); 1096 1097 if (++work_done == work_to_do) 1098 break; 1099 1100 cq_desc = &cq_desc_base[cq->tail_idx]; 1101 } 1102 1103 /* Update the queue indices and ring the doorbell */ 1104 if (work_done) 1105 ionic_q_flush(q); 1106 } 1107 1108 /* 1109 * Stop Receive Units for specified queue. 1110 */ 1111 int __rte_cold 1112 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1113 { 1114 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; 1115 struct ionic_rx_stats *stats; 1116 struct ionic_rx_qcq *rxq; 1117 1118 IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); 1119 1120 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1121 1122 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1123 1124 ionic_lif_rxq_deinit(rxq); 1125 1126 /* Free all buffers from descriptor ring */ 1127 ionic_rx_empty(rxq); 1128 1129 stats = &rxq->stats; 1130 IONIC_PRINT(DEBUG, "RX queue %u pkts %ju mtod %ju", 1131 rxq->qcq.q.index, stats->packets, stats->mtods); 1132 1133 return 0; 1134 } 1135 1136 uint16_t 1137 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1138 uint16_t nb_pkts) 1139 { 1140 struct ionic_rx_qcq *rxq = rx_queue; 1141 struct ionic_rx_service rx_svc; 1142 1143 rx_svc.rx_pkts = rx_pkts; 1144 rx_svc.nb_rx = 0; 1145 1146 ionic_rxq_service(rxq, nb_pkts, &rx_svc); 1147 1148 return rx_svc.nb_rx; 1149 } 1150