1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2022 Advanced Micro Devices, Inc. 3 */ 4 5 #include <stdio.h> 6 #include <errno.h> 7 #include <stdint.h> 8 #include <assert.h> 9 10 #include <rte_common.h> 11 #include <rte_byteorder.h> 12 #include <rte_atomic.h> 13 #include <rte_mempool.h> 14 #include <rte_mbuf.h> 15 #include <rte_ether.h> 16 #include <rte_prefetch.h> 17 18 #include "ionic.h" 19 #include "ionic_if.h" 20 #include "ionic_dev.h" 21 #include "ionic_lif.h" 22 #include "ionic_rxtx.h" 23 24 static __rte_always_inline void 25 ionic_tx_flush(struct ionic_tx_qcq *txq) 26 { 27 struct ionic_cq *cq = &txq->qcq.cq; 28 struct ionic_queue *q = &txq->qcq.q; 29 struct rte_mbuf *txm; 30 struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base; 31 void **info; 32 33 cq_desc = &cq_desc_base[cq->tail_idx]; 34 35 while (color_match(cq_desc->color, cq->done_color)) { 36 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 37 if (cq->tail_idx == 0) 38 cq->done_color = !cq->done_color; 39 40 /* Prefetch 4 x 16B comp at cq->tail_idx + 4 */ 41 if ((cq->tail_idx & 0x3) == 0) 42 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 43 44 while (q->tail_idx != rte_le_to_cpu_16(cq_desc->comp_index)) { 45 /* Prefetch 8 mbuf ptrs at q->tail_idx + 2 */ 46 rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 2)]); 47 48 /* Prefetch next mbuf */ 49 void **next_info = 50 &q->info[Q_NEXT_TO_SRVC(q, 1)]; 51 if (next_info[0]) 52 rte_mbuf_prefetch_part2(next_info[0]); 53 54 info = &q->info[q->tail_idx]; 55 { 56 txm = info[0]; 57 58 if (txq->flags & IONIC_QCQ_F_FAST_FREE) 59 rte_mempool_put(txm->pool, txm); 60 else 61 rte_pktmbuf_free_seg(txm); 62 63 info[0] = NULL; 64 } 65 66 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 67 } 68 69 cq_desc = &cq_desc_base[cq->tail_idx]; 70 } 71 } 72 73 static __rte_always_inline int 74 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm) 75 { 76 struct ionic_queue *q = &txq->qcq.q; 77 struct ionic_txq_desc *desc, *desc_base = q->base; 78 struct ionic_tx_stats *stats = &txq->stats; 79 void **info; 80 uint64_t ol_flags = txm->ol_flags; 81 uint64_t addr, cmd; 82 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 83 uint8_t flags = 0; 84 85 if (txm->nb_segs > 1) 86 return -EINVAL; 87 88 desc = &desc_base[q->head_idx]; 89 info = &q->info[q->head_idx]; 90 91 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM) && 92 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 93 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 94 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 95 } 96 97 if (((ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) && 98 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 99 ((ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) && 100 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 101 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 102 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 103 } 104 105 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 106 stats->no_csum++; 107 108 if (((ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) || 109 (ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM)) && 110 ((ol_flags & RTE_MBUF_F_TX_OUTER_IPV4) || 111 (ol_flags & RTE_MBUF_F_TX_OUTER_IPV6))) { 112 flags |= IONIC_TXQ_DESC_FLAG_ENCAP; 113 } 114 115 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 116 flags |= IONIC_TXQ_DESC_FLAG_VLAN; 117 desc->vlan_tci = rte_cpu_to_le_16(txm->vlan_tci); 118 } 119 120 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 121 122 cmd = encode_txq_desc_cmd(opcode, flags, 0, addr); 123 desc->cmd = rte_cpu_to_le_64(cmd); 124 desc->len = rte_cpu_to_le_16(txm->data_len); 125 126 info[0] = txm; 127 128 q->head_idx = Q_NEXT_TO_POST(q, 1); 129 130 return 0; 131 } 132 133 uint16_t 134 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 135 uint16_t nb_pkts) 136 { 137 struct ionic_tx_qcq *txq = tx_queue; 138 struct ionic_queue *q = &txq->qcq.q; 139 struct ionic_tx_stats *stats = &txq->stats; 140 struct rte_mbuf *mbuf; 141 uint32_t bytes_tx = 0; 142 uint16_t nb_avail, nb_tx = 0; 143 int err; 144 145 struct ionic_txq_desc *desc_base = q->base; 146 if (!(txq->flags & IONIC_QCQ_F_CMB)) 147 rte_prefetch0(&desc_base[q->head_idx]); 148 rte_prefetch0(&q->info[q->head_idx]); 149 150 if (tx_pkts) { 151 rte_mbuf_prefetch_part1(tx_pkts[0]); 152 rte_mbuf_prefetch_part2(tx_pkts[0]); 153 } 154 155 if (ionic_q_space_avail(q) < txq->free_thresh) { 156 /* Cleaning old buffers */ 157 ionic_tx_flush(txq); 158 } 159 160 nb_avail = ionic_q_space_avail(q); 161 if (nb_avail < nb_pkts) { 162 stats->stop += nb_pkts - nb_avail; 163 nb_pkts = nb_avail; 164 } 165 166 while (nb_tx < nb_pkts) { 167 uint16_t next_idx = Q_NEXT_TO_POST(q, 1); 168 if (!(txq->flags & IONIC_QCQ_F_CMB)) 169 rte_prefetch0(&desc_base[next_idx]); 170 rte_prefetch0(&q->info[next_idx]); 171 172 if (nb_tx + 1 < nb_pkts) { 173 rte_mbuf_prefetch_part1(tx_pkts[nb_tx + 1]); 174 rte_mbuf_prefetch_part2(tx_pkts[nb_tx + 1]); 175 } 176 177 mbuf = tx_pkts[nb_tx]; 178 179 if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 180 err = ionic_tx_tso(txq, mbuf); 181 else 182 err = ionic_tx(txq, mbuf); 183 if (err) { 184 stats->drop += nb_pkts - nb_tx; 185 break; 186 } 187 188 bytes_tx += mbuf->pkt_len; 189 nb_tx++; 190 } 191 192 if (nb_tx > 0) { 193 rte_wmb(); 194 ionic_q_flush(q); 195 196 stats->packets += nb_tx; 197 stats->bytes += bytes_tx; 198 } 199 200 return nb_tx; 201 } 202 203 /* 204 * Cleans one descriptor. Connects the filled mbufs into a chain. 205 * Does not advance the tail index. 206 */ 207 static __rte_always_inline void 208 ionic_rx_clean_one(struct ionic_rx_qcq *rxq, 209 struct ionic_rxq_comp *cq_desc, 210 struct ionic_rx_service *rx_svc) 211 { 212 struct ionic_queue *q = &rxq->qcq.q; 213 struct rte_mbuf *rxm; 214 struct ionic_rx_stats *stats = &rxq->stats; 215 uint64_t pkt_flags = 0; 216 uint32_t pkt_type; 217 uint16_t cq_desc_len; 218 uint8_t ptype, cflags; 219 void **info; 220 221 cq_desc_len = rte_le_to_cpu_16(cq_desc->len); 222 223 info = &q->info[q->tail_idx]; 224 225 rxm = info[0]; 226 227 if (cq_desc->status) { 228 stats->bad_cq_status++; 229 return; 230 } 231 232 if (cq_desc_len > rxq->frame_size || cq_desc_len == 0) { 233 stats->bad_len++; 234 return; 235 } 236 237 info[0] = NULL; 238 239 /* Set the mbuf metadata based on the cq entry */ 240 rxm->rearm_data[0] = rxq->rearm_data; 241 rxm->pkt_len = cq_desc_len; 242 rxm->data_len = cq_desc_len; 243 244 /* RSS */ 245 pkt_flags |= RTE_MBUF_F_RX_RSS_HASH; 246 rxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash); 247 248 /* Vlan Strip */ 249 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 250 pkt_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED; 251 rxm->vlan_tci = rte_le_to_cpu_16(cq_desc->vlan_tci); 252 } 253 254 /* Checksum */ 255 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 256 cflags = cq_desc->csum_flags & IONIC_CSUM_FLAG_MASK; 257 pkt_flags |= ionic_csum_flags[cflags]; 258 } 259 260 rxm->ol_flags = pkt_flags; 261 262 /* Packet Type */ 263 ptype = cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK; 264 pkt_type = ionic_ptype_table[ptype]; 265 if (pkt_type == RTE_PTYPE_UNKNOWN) { 266 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 267 struct rte_ether_hdr *); 268 uint16_t ether_type = eth_h->ether_type; 269 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 270 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 271 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_LLDP)) 272 pkt_type = RTE_PTYPE_L2_ETHER_LLDP; 273 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_1588)) 274 pkt_type = RTE_PTYPE_L2_ETHER_TIMESYNC; 275 stats->mtods++; 276 } else if (pkt_flags & RTE_MBUF_F_RX_VLAN) { 277 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN; 278 } else { 279 pkt_type |= RTE_PTYPE_L2_ETHER; 280 } 281 282 rxm->packet_type = pkt_type; 283 284 rx_svc->rx_pkts[rx_svc->nb_rx] = rxm; 285 rx_svc->nb_rx++; 286 287 stats->packets++; 288 stats->bytes += rxm->pkt_len; 289 } 290 291 /* 292 * Fills one descriptor with mbufs. Does not advance the head index. 293 */ 294 static __rte_always_inline int 295 ionic_rx_fill_one(struct ionic_rx_qcq *rxq) 296 { 297 struct ionic_queue *q = &rxq->qcq.q; 298 struct rte_mbuf *rxm; 299 struct ionic_rxq_desc *desc, *desc_base = q->base; 300 rte_iova_t data_iova; 301 void **info; 302 int ret; 303 304 info = &q->info[q->head_idx]; 305 desc = &desc_base[q->head_idx]; 306 307 /* mbuf is unused */ 308 if (info[0]) 309 return 0; 310 311 if (rxq->mb_idx == 0) { 312 ret = rte_mempool_get_bulk(rxq->mb_pool, 313 (void **)rxq->mbs, 314 IONIC_MBUF_BULK_ALLOC); 315 if (ret) { 316 assert(0); 317 return -ENOMEM; 318 } 319 320 rxq->mb_idx = IONIC_MBUF_BULK_ALLOC; 321 } 322 323 rxm = rxq->mbs[--rxq->mb_idx]; 324 info[0] = rxm; 325 326 data_iova = rte_mbuf_data_iova_default(rxm); 327 desc->addr = rte_cpu_to_le_64(data_iova); 328 329 return 0; 330 } 331 332 /* 333 * Walk the CQ to find completed receive descriptors. 334 * Any completed descriptor found is refilled. 335 */ 336 static __rte_always_inline void 337 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 338 struct ionic_rx_service *rx_svc) 339 { 340 struct ionic_cq *cq = &rxq->qcq.cq; 341 struct ionic_queue *q = &rxq->qcq.q; 342 struct ionic_rxq_desc *q_desc_base = q->base; 343 struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; 344 uint32_t work_done = 0; 345 346 cq_desc = &cq_desc_base[cq->tail_idx]; 347 348 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 349 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 350 if (cq->tail_idx == 0) 351 cq->done_color = !cq->done_color; 352 353 /* Prefetch 8 x 8B bufinfo */ 354 rte_prefetch0(&q->info[Q_NEXT_TO_SRVC(q, 8)]); 355 /* Prefetch 4 x 16B comp */ 356 rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]); 357 /* Prefetch 4 x 16B descriptors */ 358 if (!(rxq->flags & IONIC_QCQ_F_CMB)) 359 rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]); 360 361 /* Clean one descriptor */ 362 ionic_rx_clean_one(rxq, cq_desc, rx_svc); 363 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 364 365 /* Fill one descriptor */ 366 (void)ionic_rx_fill_one(rxq); 367 368 q->head_idx = Q_NEXT_TO_POST(q, 1); 369 370 if (++work_done == work_to_do) 371 break; 372 373 cq_desc = &cq_desc_base[cq->tail_idx]; 374 } 375 376 /* Update the queue indices and ring the doorbell */ 377 if (work_done) 378 ionic_q_flush(q); 379 } 380 381 uint16_t 382 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 383 uint16_t nb_pkts) 384 { 385 struct ionic_rx_qcq *rxq = rx_queue; 386 struct ionic_rx_service rx_svc; 387 388 rx_svc.rx_pkts = rx_pkts; 389 rx_svc.nb_rx = 0; 390 391 ionic_rxq_service(rxq, nb_pkts, &rx_svc); 392 393 return rx_svc.nb_rx; 394 } 395 396 /* 397 * Fills all descriptors with mbufs. 398 */ 399 int __rte_cold 400 ionic_rx_fill(struct ionic_rx_qcq *rxq) 401 { 402 struct ionic_queue *q = &rxq->qcq.q; 403 uint32_t i; 404 int err = 0; 405 406 for (i = 0; i < q->num_descs - 1u; i++) { 407 err = ionic_rx_fill_one(rxq); 408 if (err) 409 break; 410 411 q->head_idx = Q_NEXT_TO_POST(q, 1); 412 } 413 414 ionic_q_flush(q); 415 416 return err; 417 } 418