1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2022 Advanced Micro Devices, Inc. 3 */ 4 5 #include <stdio.h> 6 #include <errno.h> 7 #include <stdint.h> 8 #include <assert.h> 9 10 #include <rte_common.h> 11 #include <rte_byteorder.h> 12 #include <rte_atomic.h> 13 #include <rte_mempool.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <rte_prefetch.h> 17 18 #include "ionic.h" 19 #include "ionic_if.h" 20 #include "ionic_dev.h" 21 #include "ionic_lif.h" 22 #include "ionic_rxtx.h" 23 24 static __rte_always_inline void 25 ionic_tx_flush(struct ionic_tx_qcq *txq) 26 { 27 struct ionic_cq *cq = &txq->qcq.cq; 28 struct ionic_queue *q = &txq->qcq.q; 29 struct ionic_tx_stats *stats = &txq->stats; 30 struct rte_mbuf *txm; 31 struct ionic_txq_comp *cq_desc_base = cq->base; 32 volatile struct ionic_txq_comp *cq_desc; 33 void **info; 34 35 cq_desc = &cq_desc_base[cq->tail_idx]; 36 37 while (color_match(cq_desc->color, cq->done_color)) { 38 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 39 if (cq->tail_idx == 0) 40 cq->done_color = !cq->done_color; 41 42 /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ 43 if ((cq->tail_idx & 0x3) == 0) 44 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 45 46 while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { 47 /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ 48 rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 2)]); 49 50 /* Prefetch next mbuf */ 51 void **next_info = 52 &q->info[Q_NEXT_TO_SRVC(q, 1)]; 53 if (next_info[0]) 54 rte_mbuf_prefetch_part2(next_info[0]); 55 56 info = &q->info[q->tail_idx]; 57 { 58 txm = info[0]; 59 60 if (txq->flags & IONIC_QCQ_F_FAST_FREE) 61 rte_mempool_put(txm->pool, txm); 62 else 63 rte_pktmbuf_free_seg(txm); 64 65 info[0] = NULL; 66 } 67 68 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 69 } 70 71 cq_desc = &cq_desc_base[cq->tail_idx]; 72 stats->comps++; 73 } 74 } 75 76 static __rte_always_inline int 77 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) 78 { 79 struct ionic_queue *q = &txq->qcq.q; 80 struct ionic_txq_desc *desc, *desc_base = q->base; 81 struct ionic_tx_stats *stats = &txq->stats; 82 void **info; 83 uint64_t ol_flags = txm->ol_flags; 84 uint64_t addr, cmd; 85 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 86 uint8_t flags = 0; 87 88 if (txm->nb_segs > 1) 89 return -EINVAL; 90 91 desc = &desc_base[q->head_idx]; 92 info = &q->info[q->head_idx]; 93 94 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && 95 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 96 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 97 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 98 } 99 100 if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && 101 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 102 ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && 103 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 104 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 105 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 106 } 107 108 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 109 stats->no_csum++; 110 111 if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || 112 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && 113 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || 114 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { 115 flags |= IONIC_TXQ_DESC_FLAG_ENCAP; 116 } 117 118 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 119 flags |= IONIC_TXQ_DESC_FLAG_VLAN; 120 desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); 121 } 122 123 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 124 125 cmd = encode_txq_desc_cmd(opcode, flags, 0, addr); 126 desc->cmd = rte_cpu_to_le_64(cmd); 127 desc->len = rte_cpu_to_le_16(txm->data_len); 128 129 info[0] = txm; 130 131 q->head_idx = Q_NEXT_TO_POST(q, 1); 132 133 return 0; 134 } 135 136 uint16_t 137 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 138 uint16_t nb_pkts) 139 { 140 struct ionic_tx_qcq *txq = tx_queue; 141 struct ionic_queue *q = &txq->qcq.q; 142 struct ionic_txq_desc *desc_base = q->base; 143 struct ionic_tx_stats *stats = &txq->stats; 144 struct rte_mbuf *mbuf; 145 uint32_t bytes_tx = 0; 146 uint16_t nb_avail, nb_tx = 0; 147 uint64_t then, now, hz, delta; 148 int err; 149 150 rte_prefetch0(&desc_base[q->head_idx]); 151 rte_prefetch0(&q->info[q->head_idx]); 152 153 if (nb_pkts) { 154 rte_mbuf_prefetch_part1(tx_pkts[0]); 155 rte_mbuf_prefetch_part2(tx_pkts[0]); 156 } 157 158 if (ionic_q_space_avail(q) < txq->free_thresh) { 159 /* Cleaning old buffers */ 160 ionic_tx_flush(txq); 161 } 162 163 nb_avail = ionic_q_space_avail(q); 164 if (nb_avail < nb_pkts) { 165 stats->stop += nb_pkts - nb_avail; 166 nb_pkts = nb_avail; 167 } 168 169 while (nb_tx < nb_pkts) { 170 uint16_t next_idx = Q_NEXT_TO_POST(q, 1); 171 rte_prefetch0(&desc_base[next_idx]); 172 rte_prefetch0(&q->info[next_idx]); 173 174 if (nb_tx + 1 < nb_pkts) { 175 rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); 176 rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); 177 } 178 179 mbuf = tx_pkts[nb_tx]; 180 181 if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 182 err = ionic_tx_tso(txq, mbuf); 183 else 184 err = ionic_tx(txq, mbuf); 185 if (err) { 186 stats->drop += nb_pkts - nb_tx; 187 break; 188 } 189 190 bytes_tx += mbuf->pkt_len; 191 nb_tx++; 192 } 193 194 if (nb_tx > 0) { 195 rte_wmb(); 196 ionic_txq_flush(q); 197 198 txq->last_wdog_cycles = rte_get_timer_cycles(); 199 200 stats->packets += nb_tx; 201 stats->bytes += bytes_tx; 202 } else { 203 /* 204 * Ring the doorbell again if no work could be posted and work 205 * is still pending after the deadline. 206 */ 207 if (q->head_idx != q->tail_idx) { 208 then = txq->last_wdog_cycles; 209 now = rte_get_timer_cycles(); 210 hz = rte_get_timer_hz(); 211 delta = (now - then) * 1000; 212 213 if (delta >= hz * IONIC_Q_WDOG_MS) { 214 ionic_q_flush(q); 215 txq->last_wdog_cycles = now; 216 } 217 } 218 } 219 220 return nb_tx; 221 } 222 223 /* 224 * Cleans one descriptor. Connects the filled mbufs into a chain. 225 * Does not advance the tail index. 226 */ 227 static __rte_always_inline void 228 ionic_rx_clean_one(struct ionic_rx_qcq *rxq, 229 volatile struct ionic_rxq_comp *cq_desc, 230 struct ionic_rx_service *rx_svc) 231 { 232 struct ionic_queue *q = &rxq->qcq.q; 233 struct rte_mbuf *rxm; 234 struct ionic_rx_stats *stats = &rxq->stats; 235 uint64_t pkt_flags = 0; 236 uint32_t pkt_type; 237 uint16_t cq_desc_len; 238 uint8_t ptype, cflags; 239 void **info; 240 241 cq_desc_len = rte_le_to_cpu_16(cq_desc->len); 242 243 info = &q->info[q->tail_idx]; 244 245 rxm = info[0]; 246 247 if (cq_desc->status) { 248 stats->bad_cq_status++; 249 return; 250 } 251 252 if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { 253 stats->bad_len++; 254 return; 255 } 256 257 info[0] = NULL; 258 259 /* Set the mbuf metadata based on the cq entry */ 260 rxm->rearm_data[0] = rxq->rearm_data; 261 rxm->pkt_len = cq_desc_len; 262 rxm->data_len = cq_desc_len; 263 264 /* RSS */ 265 pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; 266 rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); 267 268 /* Vlan Strip */ 269 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 270 pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 271 rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); 272 } 273 274 /* Checksum */ 275 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 276 cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; 277 pkt_flags |= ionic_csum_flags[cflags]; 278 } 279 280 rxm->ol_flags = pkt_flags; 281 282 /* Packet Type */ 283 ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; 284 pkt_type = ionic_ptype_table[ptype]; 285 if (pkt_type == RTE_PTYPE_UNKNOWN) { 286 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 287 struct rte_ether_hdr *); 288 uint16_t ether_type = eth_h->ether_type; 289 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 290 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 291 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) 292 pkt_type = RTE_PTYPE_L2_ETHER_LLDP; 293 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) 294 pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; 295 stats->mtods++; 296 } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) { 297 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 298 } else { 299 pkt_type |= RTE_PTYPE_L2_ETHER; 300 } 301 302 rxm->packet_type = pkt_type; 303 304 rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; 305 rx_svc->nb_rx++; 306 307 stats->packets++; 308 stats->bytes += rxm->pkt_len; 309 } 310 311 /* 312 * Fills one descriptor with mbufs. Does not advance the head index. 313 */ 314 static __rte_always_inline int 315 ionic_rx_fill_one(struct ionic_rx_qcq *rxq) 316 { 317 struct ionic_queue *q = &rxq->qcq.q; 318 struct rte_mbuf *rxm; 319 struct ionic_rxq_desc *desc, *desc_base = q->base; 320 rte_iova_t data_iova; 321 void **info; 322 int ret; 323 324 info = &q->info[q->head_idx]; 325 desc = &desc_base[q->head_idx]; 326 327 /* mbuf is unused */ 328 if (info[0]) 329 return 0; 330 331 if (rxq->mb_idx == 0) { 332 ret = rte_mempool_get_bulk(rxq->mb_pool, 333 (void **)rxq->mbs, 334 IONIC_MBUF_BULK_ALLOC); 335 if (ret) { 336 assert(0); 337 return -ENOMEM; 338 } 339 340 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; 341 } 342 343 rxm = rxq->mbs[--rxq->mb_idx]; 344 info[0] = rxm; 345 346 data_iova = rte_mbuf_data_iova_default(rxm); 347 desc->addr = rte_cpu_to_le_64(data_iova); 348 349 return 0; 350 } 351 352 /* 353 * Walk the CQ to find completed receive descriptors. 354 * Any completed descriptor found is refilled. 355 */ 356 static __rte_always_inline void 357 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 358 struct ionic_rx_service *rx_svc) 359 { 360 struct ionic_cq *cq = &rxq->qcq.cq; 361 struct ionic_queue *q = &rxq->qcq.q; 362 struct ionic_rxq_desc *q_desc_base = q->base; 363 struct ionic_rxq_comp *cq_desc_base = cq->base; 364 volatile struct ionic_rxq_comp *cq_desc; 365 uint32_t work_done = 0; 366 uint64_t then, now, hz, delta; 367 368 cq_desc = &cq_desc_base[cq->tail_idx]; 369 370 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 371 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 372 if (cq->tail_idx == 0) 373 cq->done_color = !cq->done_color; 374 375 /* Prefetch 8 x 8B bufinfo */ 376 rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 8)]); 377 /* Prefetch 4 x 16B comp */ 378 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 379 /* Prefetch 4 x 16B descriptors */ 380 rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); 381 382 /* Clean one descriptor */ 383 ionic_rx_clean_one(rxq, cq_desc, rx_svc); 384 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 385 386 /* Fill one descriptor */ 387 (void)ionic_rx_fill_one(rxq); 388 389 q->head_idx = Q_NEXT_TO_POST(q, 1); 390 391 if (++work_done == work_to_do) 392 break; 393 394 cq_desc = &cq_desc_base[cq->tail_idx]; 395 } 396 397 /* Update the queue indices and ring the doorbell */ 398 if (work_done) { 399 ionic_rxq_flush(q); 400 401 rxq->last_wdog_cycles = rte_get_timer_cycles(); 402 rxq->wdog_ms = IONIC_Q_WDOG_MS; 403 } else { 404 /* 405 * Ring the doorbell again if no recvs were posted and the 406 * recv queue is not empty after the deadline. 407 * 408 * Exponentially back off the deadline to avoid excessive 409 * doorbells when the recv queue is idle. 410 */ 411 if (q->head_idx != q->tail_idx) { 412 then = rxq->last_wdog_cycles; 413 now = rte_get_timer_cycles(); 414 hz = rte_get_timer_hz(); 415 delta = (now - then) * 1000; 416 417 if (delta >= hz * rxq->wdog_ms) { 418 ionic_q_flush(q); 419 rxq->last_wdog_cycles = now; 420 421 delta = 2 * rxq->wdog_ms; 422 if (delta > IONIC_Q_WDOG_MAX_MS) 423 delta = IONIC_Q_WDOG_MAX_MS; 424 425 rxq->wdog_ms = delta; 426 } 427 } 428 } 429 } 430 431 uint16_t 432 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 433 uint16_t nb_pkts) 434 { 435 struct ionic_rx_qcq *rxq = rx_queue; 436 struct ionic_rx_service rx_svc; 437 438 rx_svc.rx_pkts = rx_pkts; 439 rx_svc.nb_rx = 0; 440 441 ionic_rxq_service(rxq, nb_pkts, &rx_svc); 442 443 return rx_svc.nb_rx; 444 } 445 446 /* 447 * Fills all descriptors with mbufs. 448 */ 449 int __rte_cold 450 ionic_rx_fill(struct ionic_rx_qcq *rxq) 451 { 452 struct ionic_queue *q = &rxq->qcq.q; 453 uint32_t i; 454 int err = 0; 455 456 for (i = 0; i < q->num_descs - 1u; i++) { 457 err = ionic_rx_fill_one(rxq); 458 if (err) 459 break; 460 461 q->head_idx = Q_NEXT_TO_POST(q, 1); 462 } 463 464 ionic_rxq_flush(q); 465 466 return err; 467 } 468