1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2022 Advanced Micro Devices, Inc. 3 */ 4 5 #include <stdio.h> 6 #include <errno.h> 7 #include <stdint.h> 8 #include <assert.h> 9 10 #include <rte_common.h> 11 #include <rte_byteorder.h> 12 #include <rte_atomic.h> 13 #include <rte_mempool.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <rte_prefetch.h> 17 18 #include "ionic.h" 19 #include "ionic_if.h" 20 #include "ionic_dev.h" 21 #include "ionic_lif.h" 22 #include "ionic_rxtx.h" 23 24 static __rte_always_inline void 25 ionic_tx_flush_sg(struct ionic_tx_qcq *txq) 26 { 27 struct ionic_cq *cq = &txq->qcq.cq; 28 struct ionic_queue *q = &txq->qcq.q; 29 struct ionic_tx_stats *stats = &txq->stats; 30 struct rte_mbuf *txm; 31 struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; 32 void **info; 33 uint32_t i; 34 35 cq_desc = &cq_desc_base[cq->tail_idx]; 36 37 while (color_match(cq_desc->color, cq->done_color)) { 38 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 39 if (cq->tail_idx == 0) 40 cq->done_color = !cq->done_color; 41 42 /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ 43 if ((cq->tail_idx & 0x3) == 0) 44 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 45 46 while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { 47 /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ 48 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 2))); 49 50 /* Prefetch next mbuf */ 51 void **next_info = 52 IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 1)); 53 if (next_info[0]) 54 rte_mbuf_prefetch_part2(next_info[0]); 55 if (next_info[1]) 56 rte_mbuf_prefetch_part2(next_info[1]); 57 58 info = IONIC_INFO_PTR(q, q->tail_idx); 59 for (i = 0; i < q->num_segs; i++) { 60 txm = info[i]; 61 if (!txm) 62 break; 63 64 if (txq->flags & IONIC_QCQ_F_FAST_FREE) 65 rte_mempool_put(txm->pool, txm); 66 else 67 rte_pktmbuf_free_seg(txm); 68 69 info[i] = NULL; 70 } 71 72 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 73 } 74 75 cq_desc = &cq_desc_base[cq->tail_idx]; 76 stats->comps++; 77 } 78 } 79 80 static __rte_always_inline int 81 ionic_tx_sg(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) 82 { 83 struct ionic_queue *q = &txq->qcq.q; 84 struct ionic_txq_desc *desc, *desc_base = q->base; 85 struct ionic_txq_sg_desc_v1 *sg_desc, *sg_desc_base = q->sg_base; 86 struct ionic_txq_sg_elem *elem; 87 struct ionic_tx_stats *stats = &txq->stats; 88 struct rte_mbuf *txm_seg; 89 rte_iova_t data_iova; 90 void **info; 91 uint64_t ol_flags = txm->ol_flags; 92 uint64_t addr, cmd; 93 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 94 uint8_t flags = 0; 95 96 desc = &desc_base[q->head_idx]; 97 sg_desc = &sg_desc_base[q->head_idx]; 98 info = IONIC_INFO_PTR(q, q->head_idx); 99 100 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && 101 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 102 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 103 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 104 } 105 106 if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && 107 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 108 ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && 109 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 110 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 111 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 112 } 113 114 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 115 stats->no_csum++; 116 117 if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || 118 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && 119 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || 120 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { 121 flags |= IONIC_TXQ_DESC_FLAG_ENCAP; 122 } 123 124 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 125 flags |= IONIC_TXQ_DESC_FLAG_VLAN; 126 desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); 127 } 128 129 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 130 131 cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 132 desc->cmd = rte_cpu_to_le_64(cmd); 133 desc->len = rte_cpu_to_le_16(txm->data_len); 134 135 info[0] = txm; 136 137 if (txm->nb_segs > 1) { 138 txm_seg = txm->next; 139 140 elem = sg_desc->elems; 141 142 while (txm_seg != NULL) { 143 /* Stash the mbuf ptr in the array */ 144 info++; 145 *info = txm_seg; 146 147 /* Configure the SGE */ 148 data_iova = rte_mbuf_data_iova(txm_seg); 149 elem->len = rte_cpu_to_le_16(txm_seg->data_len); 150 elem->addr = rte_cpu_to_le_64(data_iova); 151 elem++; 152 153 txm_seg = txm_seg->next; 154 } 155 } 156 157 q->head_idx = Q_NEXT_TO_POST(q, 1); 158 159 return 0; 160 } 161 162 uint16_t 163 ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts, 164 uint16_t nb_pkts) 165 { 166 struct ionic_tx_qcq *txq = tx_queue; 167 struct ionic_queue *q = &txq->qcq.q; 168 struct ionic_tx_stats *stats = &txq->stats; 169 struct rte_mbuf *mbuf; 170 uint32_t bytes_tx = 0; 171 uint16_t nb_avail, nb_tx = 0; 172 uint64_t then, now, hz, delta; 173 int err; 174 175 struct ionic_txq_desc *desc_base = q->base; 176 if (!(txq->flags & IONIC_QCQ_F_CMB)) 177 rte_prefetch0(&desc_base[q->head_idx]); 178 rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx)); 179 180 if (nb_pkts) { 181 rte_mbuf_prefetch_part1(tx_pkts[0]); 182 rte_mbuf_prefetch_part2(tx_pkts[0]); 183 } 184 185 if (ionic_q_space_avail(q) < txq->free_thresh) { 186 /* Cleaning old buffers */ 187 ionic_tx_flush_sg(txq); 188 } 189 190 nb_avail = ionic_q_space_avail(q); 191 if (nb_avail < nb_pkts) { 192 stats->stop += nb_pkts - nb_avail; 193 nb_pkts = nb_avail; 194 } 195 196 while (nb_tx < nb_pkts) { 197 uint16_t next_idx = Q_NEXT_TO_POST(q, 1); 198 if (!(txq->flags & IONIC_QCQ_F_CMB)) 199 rte_prefetch0(&desc_base[next_idx]); 200 rte_prefetch0(IONIC_INFO_PTR(q, next_idx)); 201 202 if (nb_tx + 1 < nb_pkts) { 203 rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); 204 rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); 205 } 206 207 mbuf = tx_pkts[nb_tx]; 208 209 if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 210 err = ionic_tx_tso(txq, mbuf); 211 else 212 err = ionic_tx_sg(txq, mbuf); 213 if (err) { 214 stats->drop += nb_pkts - nb_tx; 215 break; 216 } 217 218 bytes_tx += mbuf->pkt_len; 219 nb_tx++; 220 } 221 222 if (nb_tx > 0) { 223 rte_wmb(); 224 ionic_q_flush(q); 225 226 txq->last_wdog_cycles = rte_get_timer_cycles(); 227 228 stats->packets += nb_tx; 229 stats->bytes += bytes_tx; 230 } else { 231 /* 232 * Ring the doorbell again if no work could be posted and work 233 * is still pending after the deadline. 234 */ 235 if (q->head_idx != q->tail_idx) { 236 then = txq->last_wdog_cycles; 237 now = rte_get_timer_cycles(); 238 hz = rte_get_timer_hz(); 239 delta = (now - then) * 1000; 240 241 if (delta >= hz * IONIC_Q_WDOG_MS) { 242 ionic_q_flush(q); 243 txq->last_wdog_cycles = now; 244 } 245 } 246 } 247 248 return nb_tx; 249 } 250 251 /* 252 * Cleans one descriptor. Connects the filled mbufs into a chain. 253 * Does not advance the tail index. 254 */ 255 static __rte_always_inline void 256 ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq, 257 struct ionic_rxq_comp *cq_desc, 258 struct ionic_rx_service *rx_svc) 259 { 260 struct ionic_queue *q = &rxq->qcq.q; 261 struct rte_mbuf *rxm; 262 struct rte_mbuf *rxm_seg, *prev_rxm; 263 struct ionic_rx_stats *stats = &rxq->stats; 264 uint64_t pkt_flags = 0; 265 uint32_t pkt_type; 266 uint32_t left, i; 267 uint16_t cq_desc_len; 268 uint8_t ptype, cflags; 269 void **info; 270 271 cq_desc_len = rte_le_to_cpu_16(cq_desc->len); 272 273 info = IONIC_INFO_PTR(q, q->tail_idx); 274 275 rxm = info[0]; 276 277 if (cq_desc->status) { 278 stats->bad_cq_status++; 279 return; 280 } 281 282 if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { 283 stats->bad_len++; 284 return; 285 } 286 287 info[0] = NULL; 288 289 /* Set the mbuf metadata based on the cq entry */ 290 rxm->rearm_data[0] = rxq->rearm_data; 291 rxm->pkt_len = cq_desc_len; 292 rxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len); 293 left = cq_desc_len - rxm->data_len; 294 rxm->nb_segs = cq_desc->num_sg_elems + 1; 295 296 prev_rxm = rxm; 297 298 for (i = 1; i < rxm->nb_segs && left; i++) { 299 rxm_seg = info[i]; 300 info[i] = NULL; 301 302 /* Set the chained mbuf metadata */ 303 rxm_seg->rearm_data[0] = rxq->rearm_seg_data; 304 rxm_seg->data_len = RTE_MIN(rxq->seg_size, left); 305 left -= rxm_seg->data_len; 306 307 /* Link the mbuf */ 308 prev_rxm->next = rxm_seg; 309 prev_rxm = rxm_seg; 310 } 311 312 /* Terminate the mbuf chain */ 313 prev_rxm->next = NULL; 314 315 /* RSS */ 316 pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; 317 rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); 318 319 /* Vlan Strip */ 320 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 321 pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 322 rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); 323 } 324 325 /* Checksum */ 326 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 327 cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; 328 pkt_flags |= ionic_csum_flags[cflags]; 329 } 330 331 rxm->ol_flags = pkt_flags; 332 333 /* Packet Type */ 334 ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; 335 pkt_type = ionic_ptype_table[ptype]; 336 if (pkt_type == RTE_PTYPE_UNKNOWN) { 337 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 338 struct rte_ether_hdr *); 339 uint16_t ether_type = eth_h->ether_type; 340 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 341 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 342 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) 343 pkt_type = RTE_PTYPE_L2_ETHER_LLDP; 344 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) 345 pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; 346 stats->mtods++; 347 } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) { 348 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 349 } else { 350 pkt_type |= RTE_PTYPE_L2_ETHER; 351 } 352 353 rxm->packet_type = pkt_type; 354 355 rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; 356 rx_svc->nb_rx++; 357 358 stats->packets++; 359 stats->bytes += rxm->pkt_len; 360 } 361 362 /* 363 * Fills one descriptor with mbufs. Does not advance the head index. 364 */ 365 static __rte_always_inline int 366 ionic_rx_fill_one_sg(struct ionic_rx_qcq *rxq) 367 { 368 struct ionic_queue *q = &rxq->qcq.q; 369 struct rte_mbuf *rxm; 370 struct rte_mbuf *rxm_seg; 371 struct ionic_rxq_desc *desc, *desc_base = q->base; 372 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; 373 rte_iova_t data_iova; 374 uint32_t i; 375 void **info; 376 int ret; 377 378 info = IONIC_INFO_PTR(q, q->head_idx); 379 desc = &desc_base[q->head_idx]; 380 sg_desc = &sg_desc_base[q->head_idx]; 381 382 /* mbuf is unused => whole chain is unused */ 383 if (info[0]) 384 return 0; 385 386 if (rxq->mb_idx == 0) { 387 ret = rte_mempool_get_bulk(rxq->mb_pool, 388 (void **)rxq->mbs, 389 IONIC_MBUF_BULK_ALLOC); 390 if (ret) { 391 assert(0); 392 return -ENOMEM; 393 } 394 395 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; 396 } 397 398 rxm = rxq->mbs[--rxq->mb_idx]; 399 info[0] = rxm; 400 401 data_iova = rte_mbuf_data_iova_default(rxm); 402 desc->addr = rte_cpu_to_le_64(data_iova); 403 404 for (i = 1; i < q->num_segs; i++) { 405 /* mbuf is unused => rest of the chain is unused */ 406 if (info[i]) 407 return 0; 408 409 if (rxq->mb_idx == 0) { 410 ret = rte_mempool_get_bulk(rxq->mb_pool, 411 (void **)rxq->mbs, 412 IONIC_MBUF_BULK_ALLOC); 413 if (ret) { 414 assert(0); 415 return -ENOMEM; 416 } 417 418 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; 419 } 420 421 rxm_seg = rxq->mbs[--rxq->mb_idx]; 422 info[i] = rxm_seg; 423 424 /* The data_off does not get set to 0 until later */ 425 data_iova = rxm_seg->buf_iova; 426 sg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova); 427 } 428 429 return 0; 430 } 431 432 /* 433 * Walk the CQ to find completed receive descriptors. 434 * Any completed descriptor found is refilled. 435 */ 436 static __rte_always_inline void 437 ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 438 struct ionic_rx_service *rx_svc) 439 { 440 struct ionic_cq *cq = &rxq->qcq.cq; 441 struct ionic_queue *q = &rxq->qcq.q; 442 struct ionic_rxq_desc *q_desc_base = q->base; 443 struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; 444 uint32_t work_done = 0; 445 uint64_t then, now, hz, delta; 446 447 cq_desc = &cq_desc_base[cq->tail_idx]; 448 449 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 450 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 451 if (cq->tail_idx == 0) 452 cq->done_color = !cq->done_color; 453 454 /* Prefetch 8 x 8B bufinfo */ 455 rte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8))); 456 /* Prefetch 4 x 16B comp */ 457 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 458 /* Prefetch 4 x 16B descriptors */ 459 if (!(rxq->flags & IONIC_QCQ_F_CMB)) 460 rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); 461 462 /* Clean one descriptor */ 463 ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc); 464 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 465 466 /* Fill one descriptor */ 467 (void)ionic_rx_fill_one_sg(rxq); 468 469 q->head_idx = Q_NEXT_TO_POST(q, 1); 470 471 if (++work_done == work_to_do) 472 break; 473 474 cq_desc = &cq_desc_base[cq->tail_idx]; 475 } 476 477 /* Update the queue indices and ring the doorbell */ 478 if (work_done) { 479 ionic_q_flush(q); 480 rxq->last_wdog_cycles = rte_get_timer_cycles(); 481 rxq->wdog_ms = IONIC_Q_WDOG_MS; 482 } else { 483 /* 484 * Ring the doorbell again if no recvs were posted and the 485 * recv queue is not empty after the deadline. 486 * 487 * Exponentially back off the deadline to avoid excessive 488 * doorbells when the recv queue is idle. 489 */ 490 if (q->head_idx != q->tail_idx) { 491 then = rxq->last_wdog_cycles; 492 now = rte_get_timer_cycles(); 493 hz = rte_get_timer_hz(); 494 delta = (now - then) * 1000; 495 496 if (delta >= hz * rxq->wdog_ms) { 497 ionic_q_flush(q); 498 rxq->last_wdog_cycles = now; 499 500 delta = 2 * rxq->wdog_ms; 501 if (delta > IONIC_Q_WDOG_MAX_MS) 502 delta = IONIC_Q_WDOG_MAX_MS; 503 504 rxq->wdog_ms = delta; 505 } 506 } 507 } 508 } 509 510 uint16_t 511 ionic_recv_pkts_sg(void *rx_queue, struct rte_mbuf **rx_pkts, 512 uint16_t nb_pkts) 513 { 514 struct ionic_rx_qcq *rxq = rx_queue; 515 struct ionic_rx_service rx_svc; 516 517 rx_svc.rx_pkts = rx_pkts; 518 rx_svc.nb_rx = 0; 519 520 ionic_rxq_service_sg(rxq, nb_pkts, &rx_svc); 521 522 return rx_svc.nb_rx; 523 } 524 525 /* 526 * Fills all descriptors with mbufs. 527 */ 528 int __rte_cold 529 ionic_rx_fill_sg(struct ionic_rx_qcq *rxq) 530 { 531 struct ionic_queue *q = &rxq->qcq.q; 532 uint32_t i; 533 int err = 0; 534 535 for (i = 0; i < q->num_descs - 1u; i++) { 536 err = ionic_rx_fill_one_sg(rxq); 537 if (err) 538 break; 539 540 q->head_idx = Q_NEXT_TO_POST(q, 1); 541 } 542 543 ionic_q_flush(q); 544 545 return err; 546 } 547