1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2022 Advanced Micro Devices, Inc. 3 */ 4 5 #include <stdio.h> 6 #include <errno.h> 7 #include <stdint.h> 8 #include <assert.h> 9 10 #include <rte_common.h> 11 #include <rte_byteorder.h> 12 #include <rte_atomic.h> 13 #include <rte_mempool.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <rte_prefetch.h> 17 18 #include "ionic.h" 19 #include "ionic_if.h" 20 #include "ionic_dev.h" 21 #include "ionic_lif.h" 22 #include "ionic_rxtx.h" 23 24 static __rte_always_inline void 25 ionic_tx_flush_sg(struct ionic_tx_qcq *txq) 26 { 27 struct ionic_cq *cq = &txq->qcq.cq; 28 struct ionic_queue *q = &txq->qcq.q; 29 struct ionic_tx_stats *stats = &txq->stats; 30 struct rte_mbuf *txm; 31 struct ionic_txq_comp *cq_desc_base = cq->base; 32 volatile struct ionic_txq_comp *cq_desc; 33 void **info; 34 uint32_t i; 35 36 cq_desc = &cq_desc_base[cq->tail_idx]; 37 38 while (color_match(cq_desc->color, cq->done_color)) { 39 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 40 if (cq->tail_idx == 0) 41 cq->done_color = !cq->done_color; 42 43 /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ 44 if ((cq->tail_idx & 0x3) == 0) 45 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 46 47 while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { 48 /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ 49 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2))); 50 51 /* Prefetch next mbuf */ 52 void **next_info = 53 IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1)); 54 if (next_info[0]) 55 rte_mbuf_prefetch_part2(next_info[0]); 56 if (next_info[1]) 57 rte_mbuf_prefetch_part2(next_info[1]); 58 59 info = IONIC_INFO_PTR(q, q->tail_idx); 60 for (i = 0; i < q->num_segs; i++) { 61 txm = info[i]; 62 if (!txm) 63 break; 64 65 if (txq->flags & IONIC_QCQ_F_FAST_FREE) 66 rte_mempool_put(txm->pool, txm); 67 else 68 rte_pktmbuf_free_seg(txm); 69 70 info[i] = NULL; 71 } 72 73 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 74 } 75 76 cq_desc = &cq_desc_base[cq->tail_idx]; 77 stats->comps++; 78 } 79 } 80 81 static __rte_always_inline int 82 ionic_tx_sg(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) 83 { 84 struct ionic_queue *q = &txq->qcq.q; 85 struct ionic_txq_desc *desc, *desc_base = q->base; 86 struct ionic_txq_sg_desc_v1 *sg_desc, *sg_desc_base = q->sg_base; 87 struct ionic_txq_sg_elem *elem; 88 struct ionic_tx_stats *stats = &txq->stats; 89 struct rte_mbuf *txm_seg; 90 rte_iova_t data_iova; 91 void **info; 92 uint64_t ol_flags = txm->ol_flags; 93 uint64_t addr, cmd; 94 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 95 uint8_t flags = 0; 96 97 desc = &desc_base[q->head_idx]; 98 sg_desc = &sg_desc_base[q->head_idx]; 99 info = IONIC_INFO_PTR(q, q->head_idx); 100 101 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && 102 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 103 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 104 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 105 } 106 107 if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && 108 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 109 ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && 110 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 111 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 112 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 113 } 114 115 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 116 stats->no_csum++; 117 118 if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || 119 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && 120 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || 121 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { 122 flags |= IONIC_TXQ_DESC_FLAG_ENCAP; 123 } 124 125 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 126 flags |= IONIC_TXQ_DESC_FLAG_VLAN; 127 desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); 128 } 129 130 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 131 132 cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 133 desc->cmd = rte_cpu_to_le_64(cmd); 134 desc->len = rte_cpu_to_le_16(txm->data_len); 135 136 info[0] = txm; 137 138 if (txm->nb_segs > 1) { 139 txm_seg = txm->next; 140 141 elem = sg_desc->elems; 142 143 while (txm_seg != NULL) { 144 /* Stash the mbuf ptr in the array */ 145 info++; 146 *info = txm_seg; 147 148 /* Configure the SGE */ 149 data_iova = rte_mbuf_data_iova(txm_seg); 150 elem->len = rte_cpu_to_le_16(txm_seg->data_len); 151 elem->addr = rte_cpu_to_le_64(data_iova); 152 elem++; 153 154 txm_seg = txm_seg->next; 155 } 156 } 157 158 q->head_idx = Q_NEXT_TO_POST(q, 1); 159 160 return 0; 161 } 162 163 uint16_t 164 ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts, 165 uint16_t nb_pkts) 166 { 167 struct ionic_tx_qcq *txq = tx_queue; 168 struct ionic_queue *q = &txq->qcq.q; 169 struct ionic_tx_stats *stats = &txq->stats; 170 struct rte_mbuf *mbuf; 171 uint32_t bytes_tx = 0; 172 uint16_t nb_avail, nb_tx = 0; 173 uint64_t then, now, hz, delta; 174 int err; 175 176 struct ionic_txq_desc *desc_base = q->base; 177 if (!(txq->flags & IONIC_QCQ_F_CMB)) 178 rte_prefetch0(&desc_base[q->head_idx]); 179 rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx)); 180 181 if (nb_pkts) { 182 rte_mbuf_prefetch_part1(tx_pkts[0]); 183 rte_mbuf_prefetch_part2(tx_pkts[0]); 184 } 185 186 if (ionic_q_space_avail(q) < txq->free_thresh) { 187 /* Cleaning old buffers */ 188 ionic_tx_flush_sg(txq); 189 } 190 191 nb_avail = ionic_q_space_avail(q); 192 if (nb_avail < nb_pkts) { 193 stats->stop += nb_pkts - nb_avail; 194 nb_pkts = nb_avail; 195 } 196 197 while (nb_tx < nb_pkts) { 198 uint16_t next_idx = Q_NEXT_TO_POST(q, 1); 199 if (!(txq->flags & IONIC_QCQ_F_CMB)) 200 rte_prefetch0(&desc_base[next_idx]); 201 rte_prefetch0(IONIC_INFO_PTR(q, next_idx)); 202 203 if (nb_tx + 1 < nb_pkts) { 204 rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); 205 rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); 206 } 207 208 mbuf = tx_pkts[nb_tx]; 209 210 if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 211 err = ionic_tx_tso(txq, mbuf); 212 else 213 err = ionic_tx_sg(txq, mbuf); 214 if (err) { 215 stats->drop += nb_pkts - nb_tx; 216 break; 217 } 218 219 bytes_tx += mbuf->pkt_len; 220 nb_tx++; 221 } 222 223 if (nb_tx > 0) { 224 rte_wmb(); 225 ionic_q_flush(q); 226 227 txq->last_wdog_cycles = rte_get_timer_cycles(); 228 229 stats->packets += nb_tx; 230 stats->bytes += bytes_tx; 231 } else { 232 /* 233 * Ring the doorbell again if no work could be posted and work 234 * is still pending after the deadline. 235 */ 236 if (q->head_idx != q->tail_idx) { 237 then = txq->last_wdog_cycles; 238 now = rte_get_timer_cycles(); 239 hz = rte_get_timer_hz(); 240 delta = (now - then) * 1000; 241 242 if (delta >= hz * IONIC_Q_WDOG_MS) { 243 ionic_q_flush(q); 244 txq->last_wdog_cycles = now; 245 } 246 } 247 } 248 249 return nb_tx; 250 } 251 252 /* 253 * Cleans one descriptor. Connects the filled mbufs into a chain. 254 * Does not advance the tail index. 255 */ 256 static __rte_always_inline void 257 ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq, 258 volatile struct ionic_rxq_comp *cq_desc, 259 struct ionic_rx_service *rx_svc) 260 { 261 struct ionic_queue *q = &rxq->qcq.q; 262 struct rte_mbuf *rxm; 263 struct rte_mbuf *rxm_seg, *prev_rxm; 264 struct ionic_rx_stats *stats = &rxq->stats; 265 uint64_t pkt_flags = 0; 266 uint32_t pkt_type; 267 uint32_t left, i; 268 uint16_t cq_desc_len; 269 uint8_t ptype, cflags; 270 void **info; 271 272 cq_desc_len = rte_le_to_cpu_16(cq_desc->len); 273 274 info = IONIC_INFO_PTR(q, q->tail_idx); 275 276 rxm = info[0]; 277 278 if (cq_desc->status) { 279 stats->bad_cq_status++; 280 return; 281 } 282 283 if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { 284 stats->bad_len++; 285 return; 286 } 287 288 info[0] = NULL; 289 290 /* Set the mbuf metadata based on the cq entry */ 291 rxm->rearm_data[0] = rxq->rearm_data; 292 rxm->pkt_len = cq_desc_len; 293 rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len); 294 left = cq_desc_len - rxm->data_len; 295 rxm->nb_segs = cq_desc->num_sg_elems + 1; 296 297 prev_rxm = rxm; 298 299 for (i = 1; i < rxm->nb_segs && left; i++) { 300 rxm_seg = info[i]; 301 info[i] = NULL; 302 303 /* Set the chained mbuf metadata */ 304 rxm_seg->rearm_data[0] = rxq->rearm_seg_data; 305 rxm_seg->data_len = RTE_MIN(rxq->seg_size, left); 306 left -= rxm_seg->data_len; 307 308 /* Link the mbuf */ 309 prev_rxm->next = rxm_seg; 310 prev_rxm = rxm_seg; 311 } 312 313 /* Terminate the mbuf chain */ 314 prev_rxm->next = NULL; 315 316 /* RSS */ 317 pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; 318 rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); 319 320 /* Vlan Strip */ 321 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 322 pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 323 rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); 324 } 325 326 /* Checksum */ 327 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 328 cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; 329 pkt_flags |= ionic_csum_flags[cflags]; 330 } 331 332 rxm->ol_flags = pkt_flags; 333 334 /* Packet Type */ 335 ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; 336 pkt_type = ionic_ptype_table[ptype]; 337 if (pkt_type == RTE_PTYPE_UNKNOWN) { 338 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 339 struct rte_ether_hdr *); 340 uint16_t ether_type = eth_h->ether_type; 341 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 342 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 343 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) 344 pkt_type = RTE_PTYPE_L2_ETHER_LLDP; 345 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) 346 pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; 347 stats->mtods++; 348 } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) { 349 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 350 } else { 351 pkt_type |= RTE_PTYPE_L2_ETHER; 352 } 353 354 rxm->packet_type = pkt_type; 355 356 rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; 357 rx_svc->nb_rx++; 358 359 stats->packets++; 360 stats->bytes += rxm->pkt_len; 361 } 362 363 /* 364 * Fills one descriptor with mbufs. Does not advance the head index. 365 */ 366 static __rte_always_inline int 367 ionic_rx_fill_one_sg(struct ionic_rx_qcq *rxq) 368 { 369 struct ionic_queue *q = &rxq->qcq.q; 370 struct rte_mbuf *rxm; 371 struct rte_mbuf *rxm_seg; 372 struct ionic_rxq_desc *desc, *desc_base = q->base; 373 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; 374 rte_iova_t data_iova; 375 uint32_t i; 376 void **info; 377 int ret; 378 379 info = IONIC_INFO_PTR(q, q->head_idx); 380 desc = &desc_base[q->head_idx]; 381 sg_desc = &sg_desc_base[q->head_idx]; 382 383 /* mbuf is unused => whole chain is unused */ 384 if (info[0]) 385 return 0; 386 387 if (rxq->mb_idx == 0) { 388 ret = rte_mempool_get_bulk(rxq->mb_pool, 389 (void **)rxq->mbs, 390 IONIC_MBUF_BULK_ALLOC); 391 if (ret) { 392 assert(0); 393 return -ENOMEM; 394 } 395 396 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; 397 } 398 399 rxm = rxq->mbs[--rxq->mb_idx]; 400 info[0] = rxm; 401 402 data_iova = rte_mbuf_data_iova_default(rxm); 403 desc->addr = rte_cpu_to_le_64(data_iova); 404 405 for (i = 1; i < q->num_segs; i++) { 406 /* mbuf is unused => rest of the chain is unused */ 407 if (info[i]) 408 return 0; 409 410 if (rxq->mb_idx == 0) { 411 ret = rte_mempool_get_bulk(rxq->mb_pool, 412 (void **)rxq->mbs, 413 IONIC_MBUF_BULK_ALLOC); 414 if (ret) { 415 assert(0); 416 return -ENOMEM; 417 } 418 419 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; 420 } 421 422 rxm_seg = rxq->mbs[--rxq->mb_idx]; 423 info[i] = rxm_seg; 424 425 /* The data_off does not get set to 0 until later */ 426 data_iova = rxm_seg->buf_iova; 427 sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova); 428 } 429 430 return 0; 431 } 432 433 /* 434 * Walk the CQ to find completed receive descriptors. 435 * Any completed descriptor found is refilled. 436 */ 437 static __rte_always_inline void 438 ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 439 struct ionic_rx_service *rx_svc) 440 { 441 struct ionic_cq *cq = &rxq->qcq.cq; 442 struct ionic_queue *q = &rxq->qcq.q; 443 struct ionic_rxq_desc *q_desc_base = q->base; 444 struct ionic_rxq_comp *cq_desc_base = cq->base; 445 volatile struct ionic_rxq_comp *cq_desc; 446 uint32_t work_done = 0; 447 uint64_t then, now, hz, delta; 448 449 cq_desc = &cq_desc_base[cq->tail_idx]; 450 451 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 452 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 453 if (cq->tail_idx == 0) 454 cq->done_color = !cq->done_color; 455 456 /* Prefetch 8 x 8B bufinfo */ 457 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8))); 458 /* Prefetch 4 x 16B comp */ 459 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 460 /* Prefetch 4 x 16B descriptors */ 461 if (!(rxq->flags & IONIC_QCQ_F_CMB)) 462 rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); 463 464 /* Clean one descriptor */ 465 ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc); 466 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 467 468 /* Fill one descriptor */ 469 (void)ionic_rx_fill_one_sg(rxq); 470 471 q->head_idx = Q_NEXT_TO_POST(q, 1); 472 473 if (++work_done == work_to_do) 474 break; 475 476 cq_desc = &cq_desc_base[cq->tail_idx]; 477 } 478 479 /* Update the queue indices and ring the doorbell */ 480 if (work_done) { 481 ionic_q_flush(q); 482 rxq->last_wdog_cycles = rte_get_timer_cycles(); 483 rxq->wdog_ms = IONIC_Q_WDOG_MS; 484 } else { 485 /* 486 * Ring the doorbell again if no recvs were posted and the 487 * recv queue is not empty after the deadline. 488 * 489 * Exponentially back off the deadline to avoid excessive 490 * doorbells when the recv queue is idle. 491 */ 492 if (q->head_idx != q->tail_idx) { 493 then = rxq->last_wdog_cycles; 494 now = rte_get_timer_cycles(); 495 hz = rte_get_timer_hz(); 496 delta = (now - then) * 1000; 497 498 if (delta >= hz * rxq->wdog_ms) { 499 ionic_q_flush(q); 500 rxq->last_wdog_cycles = now; 501 502 delta = 2 * rxq->wdog_ms; 503 if (delta > IONIC_Q_WDOG_MAX_MS) 504 delta = IONIC_Q_WDOG_MAX_MS; 505 506 rxq->wdog_ms = delta; 507 } 508 } 509 } 510 } 511 512 uint16_t 513 ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts, 514 uint16_t nb_pkts) 515 { 516 struct ionic_rx_qcq *rxq = rx_queue; 517 struct ionic_rx_service rx_svc; 518 519 rx_svc.rx_pkts = rx_pkts; 520 rx_svc.nb_rx = 0; 521 522 ionic_rxq_service_sg(rxq, nb_pkts, &rx_svc); 523 524 return rx_svc.nb_rx; 525 } 526 527 /* 528 * Fills all descriptors with mbufs. 529 */ 530 int __rte_cold 531 ionic_rx_fill_sg(struct ionic_rx_qcq *rxq) 532 { 533 struct ionic_queue *q = &rxq->qcq.q; 534 uint32_t i; 535 int err = 0; 536 537 for (i = 0; i < q->num_descs - 1u; i++) { 538 err = ionic_rx_fill_one_sg(rxq); 539 if (err) 540 break; 541 542 q->head_idx = Q_NEXT_TO_POST(q, 1); 543 } 544 545 ionic_q_flush(q); 546 547 return err; 548 } 549