1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 Cavium, Inc 3 */ 4 5 #include <unistd.h> 6 #include <stdint.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 10 #include <rte_atomic.h> 11 #include <rte_branch_prediction.h> 12 #include <rte_byteorder.h> 13 #include <rte_common.h> 14 #include <rte_cycles.h> 15 #include <rte_errno.h> 16 #include <rte_ethdev_driver.h> 17 #include <rte_ether.h> 18 #include <rte_log.h> 19 #include <rte_mbuf.h> 20 #include <rte_prefetch.h> 21 22 #include "base/nicvf_plat.h" 23 24 #include "nicvf_ethdev.h" 25 #include "nicvf_rxtx.h" 26 #include "nicvf_logs.h" 27 28 static inline void __hot 29 fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt) 30 { 31 /* Local variable sqe to avoid read from sq desc memory*/ 32 union sq_entry_t sqe; 33 uint64_t ol_flags; 34 35 /* Fill SQ header descriptor */ 36 sqe.buff[0] = 0; 37 sqe.hdr.subdesc_type = SQ_DESC_TYPE_HEADER; 38 /* Number of sub-descriptors following this one */ 39 sqe.hdr.subdesc_cnt = pkt->nb_segs; 40 sqe.hdr.tot_len = pkt->pkt_len; 41 42 ol_flags = pkt->ol_flags & NICVF_TX_OFFLOAD_MASK; 43 if (unlikely(ol_flags)) { 44 /* L4 cksum */ 45 uint64_t l4_flags = ol_flags & PKT_TX_L4_MASK; 46 if (l4_flags == PKT_TX_TCP_CKSUM) 47 sqe.hdr.csum_l4 = SEND_L4_CSUM_TCP; 48 else if (l4_flags == PKT_TX_UDP_CKSUM) 49 sqe.hdr.csum_l4 = SEND_L4_CSUM_UDP; 50 else 51 sqe.hdr.csum_l4 = SEND_L4_CSUM_DISABLE; 52 53 sqe.hdr.l3_offset = pkt->l2_len; 54 sqe.hdr.l4_offset = pkt->l3_len + pkt->l2_len; 55 56 /* L3 cksum */ 57 if (ol_flags & PKT_TX_IP_CKSUM) 58 sqe.hdr.csum_l3 = 1; 59 } 60 61 entry->buff[0] = sqe.buff[0]; 62 } 63 64 void __hot 65 nicvf_single_pool_free_xmited_buffers(struct nicvf_txq *sq) 66 { 67 int j = 0; 68 uint32_t curr_head; 69 uint32_t head = sq->head; 70 struct rte_mbuf **txbuffs = sq->txbuffs; 71 void *obj_p[NICVF_MAX_TX_FREE_THRESH] __rte_cache_aligned; 72 73 curr_head = nicvf_addr_read(sq->sq_head) >> 4; 74 while (head != curr_head) { 75 if (txbuffs[head]) 76 obj_p[j++] = txbuffs[head]; 77 78 head = (head + 1) & sq->qlen_mask; 79 } 80 81 rte_mempool_put_bulk(sq->pool, obj_p, j); 82 sq->head = curr_head; 83 sq->xmit_bufs -= j; 84 NICVF_TX_ASSERT(sq->xmit_bufs >= 0); 85 } 86 87 void __hot 88 nicvf_multi_pool_free_xmited_buffers(struct nicvf_txq *sq) 89 { 90 uint32_t n = 0; 91 uint32_t curr_head; 92 uint32_t head = sq->head; 93 struct rte_mbuf **txbuffs = sq->txbuffs; 94 95 curr_head = nicvf_addr_read(sq->sq_head) >> 4; 96 while (head != curr_head) { 97 if (txbuffs[head]) { 98 rte_pktmbuf_free_seg(txbuffs[head]); 99 n++; 100 } 101 102 head = (head + 1) & sq->qlen_mask; 103 } 104 105 sq->head = curr_head; 106 sq->xmit_bufs -= n; 107 NICVF_TX_ASSERT(sq->xmit_bufs >= 0); 108 } 109 110 static inline uint32_t __hot 111 nicvf_free_tx_desc(struct nicvf_txq *sq) 112 { 113 return ((sq->head - sq->tail - 1) & sq->qlen_mask); 114 } 115 116 /* Send Header + Packet */ 117 #define TX_DESC_PER_PKT 2 118 119 static inline uint32_t __hot 120 nicvf_free_xmitted_buffers(struct nicvf_txq *sq, struct rte_mbuf **tx_pkts, 121 uint16_t nb_pkts) 122 { 123 uint32_t free_desc = nicvf_free_tx_desc(sq); 124 125 if (free_desc < nb_pkts * TX_DESC_PER_PKT || 126 sq->xmit_bufs > sq->tx_free_thresh) { 127 if (unlikely(sq->pool == NULL)) 128 sq->pool = tx_pkts[0]->pool; 129 130 sq->pool_free(sq); 131 /* Freed now, let see the number of free descs again */ 132 free_desc = nicvf_free_tx_desc(sq); 133 } 134 return free_desc; 135 } 136 137 uint16_t __hot 138 nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 139 { 140 int i; 141 uint32_t free_desc; 142 uint32_t tail; 143 struct nicvf_txq *sq = tx_queue; 144 union sq_entry_t *desc_ptr = sq->desc; 145 struct rte_mbuf **txbuffs = sq->txbuffs; 146 struct rte_mbuf *pkt; 147 uint32_t qlen_mask = sq->qlen_mask; 148 149 tail = sq->tail; 150 free_desc = nicvf_free_xmitted_buffers(sq, tx_pkts, nb_pkts); 151 152 for (i = 0; i < nb_pkts && (int)free_desc >= TX_DESC_PER_PKT; i++) { 153 pkt = tx_pkts[i]; 154 155 txbuffs[tail] = NULL; 156 fill_sq_desc_header(desc_ptr + tail, pkt); 157 tail = (tail + 1) & qlen_mask; 158 159 txbuffs[tail] = pkt; 160 fill_sq_desc_gather(desc_ptr + tail, pkt); 161 tail = (tail + 1) & qlen_mask; 162 free_desc -= TX_DESC_PER_PKT; 163 } 164 165 sq->tail = tail; 166 sq->xmit_bufs += i; 167 rte_wmb(); 168 169 /* Inform HW to xmit the packets */ 170 nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT); 171 return i; 172 } 173 174 uint16_t __hot 175 nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts, 176 uint16_t nb_pkts) 177 { 178 int i, k; 179 uint32_t used_desc, next_used_desc, used_bufs, free_desc, tail; 180 struct nicvf_txq *sq = tx_queue; 181 union sq_entry_t *desc_ptr = sq->desc; 182 struct rte_mbuf **txbuffs = sq->txbuffs; 183 struct rte_mbuf *pkt, *seg; 184 uint32_t qlen_mask = sq->qlen_mask; 185 uint16_t nb_segs; 186 187 tail = sq->tail; 188 used_desc = 0; 189 used_bufs = 0; 190 191 free_desc = nicvf_free_xmitted_buffers(sq, tx_pkts, nb_pkts); 192 193 for (i = 0; i < nb_pkts; i++) { 194 pkt = tx_pkts[i]; 195 196 nb_segs = pkt->nb_segs; 197 198 next_used_desc = used_desc + nb_segs + 1; 199 if (next_used_desc > free_desc) 200 break; 201 used_desc = next_used_desc; 202 used_bufs += nb_segs; 203 204 txbuffs[tail] = NULL; 205 fill_sq_desc_header(desc_ptr + tail, pkt); 206 tail = (tail + 1) & qlen_mask; 207 208 txbuffs[tail] = pkt; 209 fill_sq_desc_gather(desc_ptr + tail, pkt); 210 tail = (tail + 1) & qlen_mask; 211 212 seg = pkt->next; 213 for (k = 1; k < nb_segs; k++) { 214 txbuffs[tail] = seg; 215 fill_sq_desc_gather(desc_ptr + tail, seg); 216 tail = (tail + 1) & qlen_mask; 217 seg = seg->next; 218 } 219 } 220 221 sq->tail = tail; 222 sq->xmit_bufs += used_bufs; 223 rte_wmb(); 224 225 /* Inform HW to xmit the packets */ 226 nicvf_addr_write(sq->sq_door, used_desc); 227 return i; 228 } 229 230 static const uint32_t ptype_table[16][16] __rte_cache_aligned = { 231 [L3_NONE][L4_NONE] = RTE_PTYPE_UNKNOWN, 232 [L3_NONE][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 233 [L3_NONE][L4_IPFRAG] = RTE_PTYPE_L4_FRAG, 234 [L3_NONE][L4_IPCOMP] = RTE_PTYPE_UNKNOWN, 235 [L3_NONE][L4_TCP] = RTE_PTYPE_L4_TCP, 236 [L3_NONE][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP, 237 [L3_NONE][L4_GRE] = RTE_PTYPE_TUNNEL_GRE, 238 [L3_NONE][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP, 239 [L3_NONE][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 240 [L3_NONE][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 241 [L3_NONE][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 242 243 [L3_IPV4][L4_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 244 [L3_IPV4][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4, 245 [L3_IPV4][L4_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG, 246 [L3_IPV4][L4_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 247 [L3_IPV4][L4_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 248 [L3_IPV4][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 249 [L3_IPV4][L4_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE, 250 [L3_IPV4][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 251 [L3_IPV4][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE, 252 [L3_IPV4][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN, 253 [L3_IPV4][L4_NVGRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 254 255 [L3_IPV4_OPT][L4_NONE] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 256 [L3_IPV4_OPT][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV4_EXT | 257 RTE_PTYPE_L3_IPV4, 258 [L3_IPV4_OPT][L4_IPFRAG] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG, 259 [L3_IPV4_OPT][L4_IPCOMP] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 260 [L3_IPV4_OPT][L4_TCP] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 261 [L3_IPV4_OPT][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 262 [L3_IPV4_OPT][L4_GRE] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE, 263 [L3_IPV4_OPT][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 264 [L3_IPV4_OPT][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV4_EXT | 265 RTE_PTYPE_TUNNEL_GENEVE, 266 [L3_IPV4_OPT][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV4_EXT | 267 RTE_PTYPE_TUNNEL_VXLAN, 268 [L3_IPV4_OPT][L4_NVGRE] = RTE_PTYPE_L3_IPV4_EXT | 269 RTE_PTYPE_TUNNEL_NVGRE, 270 271 [L3_IPV6][L4_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 272 [L3_IPV6][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4, 273 [L3_IPV6][L4_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG, 274 [L3_IPV6][L4_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 275 [L3_IPV6][L4_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 276 [L3_IPV6][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 277 [L3_IPV6][L4_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE, 278 [L3_IPV6][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 279 [L3_IPV6][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE, 280 [L3_IPV6][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN, 281 [L3_IPV6][L4_NVGRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_NVGRE, 282 283 [L3_IPV6_OPT][L4_NONE] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 284 [L3_IPV6_OPT][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV6_EXT | 285 RTE_PTYPE_L3_IPV4, 286 [L3_IPV6_OPT][L4_IPFRAG] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG, 287 [L3_IPV6_OPT][L4_IPCOMP] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 288 [L3_IPV6_OPT][L4_TCP] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 289 [L3_IPV6_OPT][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 290 [L3_IPV6_OPT][L4_GRE] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE, 291 [L3_IPV6_OPT][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 292 [L3_IPV6_OPT][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV6_EXT | 293 RTE_PTYPE_TUNNEL_GENEVE, 294 [L3_IPV6_OPT][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV6_EXT | 295 RTE_PTYPE_TUNNEL_VXLAN, 296 [L3_IPV6_OPT][L4_NVGRE] = RTE_PTYPE_L3_IPV6_EXT | 297 RTE_PTYPE_TUNNEL_NVGRE, 298 299 [L3_ET_STOP][L4_NONE] = RTE_PTYPE_UNKNOWN, 300 [L3_ET_STOP][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 301 [L3_ET_STOP][L4_IPFRAG] = RTE_PTYPE_L4_FRAG, 302 [L3_ET_STOP][L4_IPCOMP] = RTE_PTYPE_UNKNOWN, 303 [L3_ET_STOP][L4_TCP] = RTE_PTYPE_L4_TCP, 304 [L3_ET_STOP][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP, 305 [L3_ET_STOP][L4_GRE] = RTE_PTYPE_TUNNEL_GRE, 306 [L3_ET_STOP][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP, 307 [L3_ET_STOP][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 308 [L3_ET_STOP][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 309 [L3_ET_STOP][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 310 311 [L3_OTHER][L4_NONE] = RTE_PTYPE_UNKNOWN, 312 [L3_OTHER][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 313 [L3_OTHER][L4_IPFRAG] = RTE_PTYPE_L4_FRAG, 314 [L3_OTHER][L4_IPCOMP] = RTE_PTYPE_UNKNOWN, 315 [L3_OTHER][L4_TCP] = RTE_PTYPE_L4_TCP, 316 [L3_OTHER][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP, 317 [L3_OTHER][L4_GRE] = RTE_PTYPE_TUNNEL_GRE, 318 [L3_OTHER][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP, 319 [L3_OTHER][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 320 [L3_OTHER][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 321 [L3_OTHER][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 322 }; 323 324 static inline uint32_t __hot 325 nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0) 326 { 327 return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type]; 328 } 329 330 static inline int __hot 331 nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill) 332 { 333 int i; 334 uint32_t ltail, next_tail; 335 struct nicvf_rbdr *rbdr = rxq->shared_rbdr; 336 uint64_t mbuf_phys_off = rxq->mbuf_phys_off; 337 struct rbdr_entry_t *desc = rbdr->desc; 338 uint32_t qlen_mask = rbdr->qlen_mask; 339 uintptr_t door = rbdr->rbdr_door; 340 void *obj_p[NICVF_MAX_RX_FREE_THRESH] __rte_cache_aligned; 341 342 if (unlikely(rte_mempool_get_bulk(rxq->pool, obj_p, to_fill) < 0)) { 343 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += 344 to_fill; 345 return 0; 346 } 347 348 NICVF_RX_ASSERT((unsigned int)to_fill <= (qlen_mask - 349 (nicvf_addr_read(rbdr->rbdr_status) & NICVF_RBDR_COUNT_MASK))); 350 351 next_tail = __atomic_fetch_add(&rbdr->next_tail, to_fill, 352 __ATOMIC_ACQUIRE); 353 ltail = next_tail; 354 for (i = 0; i < to_fill; i++) { 355 struct rbdr_entry_t *entry = desc + (ltail & qlen_mask); 356 357 entry->full_addr = nicvf_mbuff_virt2phy((uintptr_t)obj_p[i], 358 mbuf_phys_off); 359 ltail++; 360 } 361 362 while (__atomic_load_n(&rbdr->tail, __ATOMIC_RELAXED) != next_tail) 363 rte_pause(); 364 365 __atomic_store_n(&rbdr->tail, ltail, __ATOMIC_RELEASE); 366 nicvf_addr_write(door, to_fill); 367 return to_fill; 368 } 369 370 static inline int32_t __hot 371 nicvf_rx_pkts_to_process(struct nicvf_rxq *rxq, uint16_t nb_pkts, 372 int32_t available_space) 373 { 374 if (unlikely(available_space < nb_pkts)) 375 rxq->available_space = nicvf_addr_read(rxq->cq_status) 376 & NICVF_CQ_CQE_COUNT_MASK; 377 378 return RTE_MIN(nb_pkts, available_space); 379 } 380 381 static inline void __hot 382 nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2, 383 struct rte_mbuf *pkt) 384 { 385 if (likely(cqe_rx_w0.rss_alg)) { 386 pkt->hash.rss = cqe_rx_w2.rss_tag; 387 pkt->ol_flags |= PKT_RX_RSS_HASH; 388 } 389 } 390 391 uint16_t __hot 392 nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 393 { 394 uint32_t i, to_process; 395 struct cqe_rx_t *cqe_rx; 396 struct rte_mbuf *pkt; 397 cqe_rx_word0_t cqe_rx_w0; 398 cqe_rx_word1_t cqe_rx_w1; 399 cqe_rx_word2_t cqe_rx_w2; 400 cqe_rx_word3_t cqe_rx_w3; 401 struct nicvf_rxq *rxq = rx_queue; 402 union cq_entry_t *desc = rxq->desc; 403 const uint64_t cqe_mask = rxq->qlen_mask; 404 uint64_t rb0_ptr, mbuf_phys_off = rxq->mbuf_phys_off; 405 const uint64_t mbuf_init = rxq->mbuf_initializer.value; 406 uint32_t cqe_head = rxq->head & cqe_mask; 407 int32_t available_space = rxq->available_space; 408 const uint8_t rbptr_offset = rxq->rbptr_offset; 409 410 to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space); 411 412 for (i = 0; i < to_process; i++) { 413 rte_prefetch_non_temporal(&desc[cqe_head + 2]); 414 cqe_rx = (struct cqe_rx_t *)&desc[cqe_head]; 415 NICVF_RX_ASSERT(((struct cq_entry_type_t *)cqe_rx)->cqe_type 416 == CQE_TYPE_RX); 417 418 NICVF_LOAD_PAIR(cqe_rx_w0.u64, cqe_rx_w1.u64, cqe_rx); 419 NICVF_LOAD_PAIR(cqe_rx_w2.u64, cqe_rx_w3.u64, &cqe_rx->word2); 420 rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset); 421 pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt 422 (rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off); 423 pkt->ol_flags = 0; 424 pkt->data_len = cqe_rx_w3.rb0_sz; 425 pkt->pkt_len = cqe_rx_w3.rb0_sz; 426 pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); 427 nicvf_mbuff_init_update(pkt, mbuf_init, cqe_rx_w1.align_pad); 428 nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); 429 rx_pkts[i] = pkt; 430 cqe_head = (cqe_head + 1) & cqe_mask; 431 nicvf_prefetch_store_keep(pkt); 432 } 433 434 if (likely(to_process)) { 435 rxq->available_space -= to_process; 436 rxq->head = cqe_head; 437 nicvf_addr_write(rxq->cq_door, to_process); 438 rxq->recv_buffers += to_process; 439 } 440 if (rxq->recv_buffers > rxq->rx_free_thresh) { 441 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rxq->rx_free_thresh); 442 NICVF_RX_ASSERT(rxq->recv_buffers >= 0); 443 } 444 445 return to_process; 446 } 447 448 static inline uint16_t __hot 449 nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, 450 uint64_t mbuf_phys_off, 451 struct rte_mbuf **rx_pkt, uint8_t rbptr_offset, 452 uint64_t mbuf_init) 453 { 454 struct rte_mbuf *pkt, *seg, *prev; 455 cqe_rx_word0_t cqe_rx_w0; 456 cqe_rx_word1_t cqe_rx_w1; 457 cqe_rx_word2_t cqe_rx_w2; 458 uint16_t *rb_sz, nb_segs, seg_idx; 459 uint64_t *rb_ptr; 460 461 NICVF_LOAD_PAIR(cqe_rx_w0.u64, cqe_rx_w1.u64, cqe_rx); 462 NICVF_RX_ASSERT(cqe_rx_w0.cqe_type == CQE_TYPE_RX); 463 cqe_rx_w2 = cqe_rx->word2; 464 rb_sz = &cqe_rx->word3.rb0_sz; 465 rb_ptr = (uint64_t *)cqe_rx + rbptr_offset; 466 nb_segs = cqe_rx_w0.rb_cnt; 467 pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt 468 (rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off); 469 470 pkt->ol_flags = 0; 471 pkt->pkt_len = cqe_rx_w1.pkt_len; 472 pkt->data_len = rb_sz[nicvf_frag_num(0)]; 473 nicvf_mbuff_init_mseg_update( 474 pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs); 475 pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); 476 nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); 477 478 *rx_pkt = pkt; 479 prev = pkt; 480 for (seg_idx = 1; seg_idx < nb_segs; seg_idx++) { 481 seg = (struct rte_mbuf *)nicvf_mbuff_phy2virt 482 (rb_ptr[seg_idx], mbuf_phys_off); 483 484 prev->next = seg; 485 seg->data_len = rb_sz[nicvf_frag_num(seg_idx)]; 486 nicvf_mbuff_init_update(seg, mbuf_init, 0); 487 488 prev = seg; 489 } 490 prev->next = NULL; 491 return nb_segs; 492 } 493 494 uint16_t __hot 495 nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, 496 uint16_t nb_pkts) 497 { 498 union cq_entry_t *cq_entry; 499 struct cqe_rx_t *cqe_rx; 500 struct nicvf_rxq *rxq = rx_queue; 501 union cq_entry_t *desc = rxq->desc; 502 const uint64_t cqe_mask = rxq->qlen_mask; 503 uint64_t mbuf_phys_off = rxq->mbuf_phys_off; 504 uint32_t i, to_process, cqe_head, buffers_consumed = 0; 505 int32_t available_space = rxq->available_space; 506 uint16_t nb_segs; 507 const uint64_t mbuf_init = rxq->mbuf_initializer.value; 508 const uint8_t rbptr_offset = rxq->rbptr_offset; 509 510 cqe_head = rxq->head & cqe_mask; 511 to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space); 512 513 for (i = 0; i < to_process; i++) { 514 rte_prefetch_non_temporal(&desc[cqe_head + 2]); 515 cq_entry = &desc[cqe_head]; 516 cqe_rx = (struct cqe_rx_t *)cq_entry; 517 nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off, 518 rx_pkts + i, rbptr_offset, mbuf_init); 519 buffers_consumed += nb_segs; 520 cqe_head = (cqe_head + 1) & cqe_mask; 521 nicvf_prefetch_store_keep(rx_pkts[i]); 522 } 523 524 if (likely(to_process)) { 525 rxq->available_space -= to_process; 526 rxq->head = cqe_head; 527 nicvf_addr_write(rxq->cq_door, to_process); 528 rxq->recv_buffers += buffers_consumed; 529 } 530 if (rxq->recv_buffers > rxq->rx_free_thresh) { 531 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rxq->rx_free_thresh); 532 NICVF_RX_ASSERT(rxq->recv_buffers >= 0); 533 } 534 535 return to_process; 536 } 537 538 uint32_t 539 nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) 540 { 541 struct nicvf_rxq *rxq; 542 543 rxq = dev->data->rx_queues[queue_idx]; 544 return nicvf_addr_read(rxq->cq_status) & NICVF_CQ_CQE_COUNT_MASK; 545 } 546 547 uint32_t 548 nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx) 549 { 550 struct nicvf_rxq *rxq; 551 uint32_t to_process; 552 uint32_t rx_free; 553 554 rxq = dev->data->rx_queues[queue_idx]; 555 to_process = rxq->recv_buffers; 556 while (rxq->recv_buffers > 0) { 557 rx_free = RTE_MIN(rxq->recv_buffers, NICVF_MAX_RX_FREE_THRESH); 558 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rx_free); 559 } 560 561 assert(rxq->recv_buffers == 0); 562 return to_process; 563 } 564