1 /* 2 * BSD LICENSE 3 * 4 * Copyright (C) Cavium networks Ltd. 2016. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Cavium networks nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <unistd.h> 34 #include <stdint.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_byteorder.h> 41 #include <rte_common.h> 42 #include <rte_cycles.h> 43 #include <rte_errno.h> 44 #include <rte_ethdev.h> 45 #include <rte_ether.h> 46 #include <rte_log.h> 47 #include <rte_mbuf.h> 48 #include <rte_prefetch.h> 49 50 #include "base/nicvf_plat.h" 51 52 #include "nicvf_ethdev.h" 53 #include "nicvf_rxtx.h" 54 #include "nicvf_logs.h" 55 56 static inline void __hot 57 fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt) 58 { 59 /* Local variable sqe to avoid read from sq desc memory*/ 60 union sq_entry_t sqe; 61 uint64_t ol_flags; 62 63 /* Fill SQ header descriptor */ 64 sqe.buff[0] = 0; 65 sqe.hdr.subdesc_type = SQ_DESC_TYPE_HEADER; 66 /* Number of sub-descriptors following this one */ 67 sqe.hdr.subdesc_cnt = pkt->nb_segs; 68 sqe.hdr.tot_len = pkt->pkt_len; 69 70 ol_flags = pkt->ol_flags & NICVF_TX_OFFLOAD_MASK; 71 if (unlikely(ol_flags)) { 72 /* L4 cksum */ 73 uint64_t l4_flags = ol_flags & PKT_TX_L4_MASK; 74 if (l4_flags == PKT_TX_TCP_CKSUM) 75 sqe.hdr.csum_l4 = SEND_L4_CSUM_TCP; 76 else if (l4_flags == PKT_TX_UDP_CKSUM) 77 sqe.hdr.csum_l4 = SEND_L4_CSUM_UDP; 78 else 79 sqe.hdr.csum_l4 = SEND_L4_CSUM_DISABLE; 80 81 sqe.hdr.l3_offset = pkt->l2_len; 82 sqe.hdr.l4_offset = pkt->l3_len + pkt->l2_len; 83 84 /* L3 cksum */ 85 if (ol_flags & PKT_TX_IP_CKSUM) 86 sqe.hdr.csum_l3 = 1; 87 } 88 89 entry->buff[0] = sqe.buff[0]; 90 } 91 92 void __hot 93 nicvf_single_pool_free_xmited_buffers(struct nicvf_txq *sq) 94 { 95 int j = 0; 96 uint32_t curr_head; 97 uint32_t head = sq->head; 98 struct rte_mbuf **txbuffs = sq->txbuffs; 99 void *obj_p[NICVF_MAX_TX_FREE_THRESH] __rte_cache_aligned; 100 101 curr_head = nicvf_addr_read(sq->sq_head) >> 4; 102 while (head != curr_head) { 103 if (txbuffs[head]) 104 obj_p[j++] = txbuffs[head]; 105 106 head = (head + 1) & sq->qlen_mask; 107 } 108 109 rte_mempool_put_bulk(sq->pool, obj_p, j); 110 sq->head = curr_head; 111 sq->xmit_bufs -= j; 112 NICVF_TX_ASSERT(sq->xmit_bufs >= 0); 113 } 114 115 void __hot 116 nicvf_multi_pool_free_xmited_buffers(struct nicvf_txq *sq) 117 { 118 uint32_t n = 0; 119 uint32_t curr_head; 120 uint32_t head = sq->head; 121 struct rte_mbuf **txbuffs = sq->txbuffs; 122 123 curr_head = nicvf_addr_read(sq->sq_head) >> 4; 124 while (head != curr_head) { 125 if (txbuffs[head]) { 126 rte_pktmbuf_free_seg(txbuffs[head]); 127 n++; 128 } 129 130 head = (head + 1) & sq->qlen_mask; 131 } 132 133 sq->head = curr_head; 134 sq->xmit_bufs -= n; 135 NICVF_TX_ASSERT(sq->xmit_bufs >= 0); 136 } 137 138 static inline uint32_t __hot 139 nicvf_free_tx_desc(struct nicvf_txq *sq) 140 { 141 return ((sq->head - sq->tail - 1) & sq->qlen_mask); 142 } 143 144 /* Send Header + Packet */ 145 #define TX_DESC_PER_PKT 2 146 147 static inline uint32_t __hot 148 nicvf_free_xmitted_buffers(struct nicvf_txq *sq, struct rte_mbuf **tx_pkts, 149 uint16_t nb_pkts) 150 { 151 uint32_t free_desc = nicvf_free_tx_desc(sq); 152 153 if (free_desc < nb_pkts * TX_DESC_PER_PKT || 154 sq->xmit_bufs > sq->tx_free_thresh) { 155 if (unlikely(sq->pool == NULL)) 156 sq->pool = tx_pkts[0]->pool; 157 158 sq->pool_free(sq); 159 /* Freed now, let see the number of free descs again */ 160 free_desc = nicvf_free_tx_desc(sq); 161 } 162 return free_desc; 163 } 164 165 uint16_t __hot 166 nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 167 { 168 int i; 169 uint32_t free_desc; 170 uint32_t tail; 171 struct nicvf_txq *sq = tx_queue; 172 union sq_entry_t *desc_ptr = sq->desc; 173 struct rte_mbuf **txbuffs = sq->txbuffs; 174 struct rte_mbuf *pkt; 175 uint32_t qlen_mask = sq->qlen_mask; 176 177 tail = sq->tail; 178 free_desc = nicvf_free_xmitted_buffers(sq, tx_pkts, nb_pkts); 179 180 for (i = 0; i < nb_pkts && (int)free_desc >= TX_DESC_PER_PKT; i++) { 181 pkt = tx_pkts[i]; 182 183 txbuffs[tail] = NULL; 184 fill_sq_desc_header(desc_ptr + tail, pkt); 185 tail = (tail + 1) & qlen_mask; 186 187 txbuffs[tail] = pkt; 188 fill_sq_desc_gather(desc_ptr + tail, pkt); 189 tail = (tail + 1) & qlen_mask; 190 free_desc -= TX_DESC_PER_PKT; 191 } 192 193 sq->tail = tail; 194 sq->xmit_bufs += i; 195 rte_wmb(); 196 197 /* Inform HW to xmit the packets */ 198 nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT); 199 return i; 200 } 201 202 uint16_t __hot 203 nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts, 204 uint16_t nb_pkts) 205 { 206 int i, k; 207 uint32_t used_desc, next_used_desc, used_bufs, free_desc, tail; 208 struct nicvf_txq *sq = tx_queue; 209 union sq_entry_t *desc_ptr = sq->desc; 210 struct rte_mbuf **txbuffs = sq->txbuffs; 211 struct rte_mbuf *pkt, *seg; 212 uint32_t qlen_mask = sq->qlen_mask; 213 uint16_t nb_segs; 214 215 tail = sq->tail; 216 used_desc = 0; 217 used_bufs = 0; 218 219 free_desc = nicvf_free_xmitted_buffers(sq, tx_pkts, nb_pkts); 220 221 for (i = 0; i < nb_pkts; i++) { 222 pkt = tx_pkts[i]; 223 224 nb_segs = pkt->nb_segs; 225 226 next_used_desc = used_desc + nb_segs + 1; 227 if (next_used_desc > free_desc) 228 break; 229 used_desc = next_used_desc; 230 used_bufs += nb_segs; 231 232 txbuffs[tail] = NULL; 233 fill_sq_desc_header(desc_ptr + tail, pkt); 234 tail = (tail + 1) & qlen_mask; 235 236 txbuffs[tail] = pkt; 237 fill_sq_desc_gather(desc_ptr + tail, pkt); 238 tail = (tail + 1) & qlen_mask; 239 240 seg = pkt->next; 241 for (k = 1; k < nb_segs; k++) { 242 txbuffs[tail] = seg; 243 fill_sq_desc_gather(desc_ptr + tail, seg); 244 tail = (tail + 1) & qlen_mask; 245 seg = seg->next; 246 } 247 } 248 249 sq->tail = tail; 250 sq->xmit_bufs += used_bufs; 251 rte_wmb(); 252 253 /* Inform HW to xmit the packets */ 254 nicvf_addr_write(sq->sq_door, used_desc); 255 return nb_pkts; 256 } 257 258 static const uint32_t ptype_table[16][16] __rte_cache_aligned = { 259 [L3_NONE][L4_NONE] = RTE_PTYPE_UNKNOWN, 260 [L3_NONE][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 261 [L3_NONE][L4_IPFRAG] = RTE_PTYPE_L4_FRAG, 262 [L3_NONE][L4_IPCOMP] = RTE_PTYPE_UNKNOWN, 263 [L3_NONE][L4_TCP] = RTE_PTYPE_L4_TCP, 264 [L3_NONE][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP, 265 [L3_NONE][L4_GRE] = RTE_PTYPE_TUNNEL_GRE, 266 [L3_NONE][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP, 267 [L3_NONE][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 268 [L3_NONE][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 269 [L3_NONE][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 270 271 [L3_IPV4][L4_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 272 [L3_IPV4][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4, 273 [L3_IPV4][L4_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG, 274 [L3_IPV4][L4_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 275 [L3_IPV4][L4_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 276 [L3_IPV4][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 277 [L3_IPV4][L4_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE, 278 [L3_IPV4][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 279 [L3_IPV4][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE, 280 [L3_IPV4][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN, 281 [L3_IPV4][L4_NVGRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 282 283 [L3_IPV4_OPT][L4_NONE] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 284 [L3_IPV4_OPT][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV4_EXT | 285 RTE_PTYPE_L3_IPV4, 286 [L3_IPV4_OPT][L4_IPFRAG] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG, 287 [L3_IPV4_OPT][L4_IPCOMP] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 288 [L3_IPV4_OPT][L4_TCP] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 289 [L3_IPV4_OPT][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 290 [L3_IPV4_OPT][L4_GRE] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE, 291 [L3_IPV4_OPT][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 292 [L3_IPV4_OPT][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV4_EXT | 293 RTE_PTYPE_TUNNEL_GENEVE, 294 [L3_IPV4_OPT][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV4_EXT | 295 RTE_PTYPE_TUNNEL_VXLAN, 296 [L3_IPV4_OPT][L4_NVGRE] = RTE_PTYPE_L3_IPV4_EXT | 297 RTE_PTYPE_TUNNEL_NVGRE, 298 299 [L3_IPV6][L4_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 300 [L3_IPV6][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4, 301 [L3_IPV6][L4_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG, 302 [L3_IPV6][L4_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 303 [L3_IPV6][L4_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 304 [L3_IPV6][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 305 [L3_IPV6][L4_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE, 306 [L3_IPV6][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 307 [L3_IPV6][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE, 308 [L3_IPV6][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN, 309 [L3_IPV6][L4_NVGRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_NVGRE, 310 311 [L3_IPV6_OPT][L4_NONE] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 312 [L3_IPV6_OPT][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV6_EXT | 313 RTE_PTYPE_L3_IPV4, 314 [L3_IPV6_OPT][L4_IPFRAG] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG, 315 [L3_IPV6_OPT][L4_IPCOMP] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 316 [L3_IPV6_OPT][L4_TCP] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 317 [L3_IPV6_OPT][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 318 [L3_IPV6_OPT][L4_GRE] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE, 319 [L3_IPV6_OPT][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 320 [L3_IPV6_OPT][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV6_EXT | 321 RTE_PTYPE_TUNNEL_GENEVE, 322 [L3_IPV6_OPT][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV6_EXT | 323 RTE_PTYPE_TUNNEL_VXLAN, 324 [L3_IPV6_OPT][L4_NVGRE] = RTE_PTYPE_L3_IPV6_EXT | 325 RTE_PTYPE_TUNNEL_NVGRE, 326 327 [L3_ET_STOP][L4_NONE] = RTE_PTYPE_UNKNOWN, 328 [L3_ET_STOP][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 329 [L3_ET_STOP][L4_IPFRAG] = RTE_PTYPE_L4_FRAG, 330 [L3_ET_STOP][L4_IPCOMP] = RTE_PTYPE_UNKNOWN, 331 [L3_ET_STOP][L4_TCP] = RTE_PTYPE_L4_TCP, 332 [L3_ET_STOP][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP, 333 [L3_ET_STOP][L4_GRE] = RTE_PTYPE_TUNNEL_GRE, 334 [L3_ET_STOP][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP, 335 [L3_ET_STOP][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 336 [L3_ET_STOP][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 337 [L3_ET_STOP][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 338 339 [L3_OTHER][L4_NONE] = RTE_PTYPE_UNKNOWN, 340 [L3_OTHER][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 341 [L3_OTHER][L4_IPFRAG] = RTE_PTYPE_L4_FRAG, 342 [L3_OTHER][L4_IPCOMP] = RTE_PTYPE_UNKNOWN, 343 [L3_OTHER][L4_TCP] = RTE_PTYPE_L4_TCP, 344 [L3_OTHER][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP, 345 [L3_OTHER][L4_GRE] = RTE_PTYPE_TUNNEL_GRE, 346 [L3_OTHER][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP, 347 [L3_OTHER][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 348 [L3_OTHER][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 349 [L3_OTHER][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 350 }; 351 352 static inline uint32_t __hot 353 nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0) 354 { 355 return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type]; 356 } 357 358 static inline int __hot 359 nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill) 360 { 361 int i; 362 uint32_t ltail, next_tail; 363 struct nicvf_rbdr *rbdr = rxq->shared_rbdr; 364 uint64_t mbuf_phys_off = rxq->mbuf_phys_off; 365 struct rbdr_entry_t *desc = rbdr->desc; 366 uint32_t qlen_mask = rbdr->qlen_mask; 367 uintptr_t door = rbdr->rbdr_door; 368 void *obj_p[NICVF_MAX_RX_FREE_THRESH] __rte_cache_aligned; 369 370 if (unlikely(rte_mempool_get_bulk(rxq->pool, obj_p, to_fill) < 0)) { 371 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += 372 to_fill; 373 return 0; 374 } 375 376 NICVF_RX_ASSERT((unsigned int)to_fill <= (qlen_mask - 377 (nicvf_addr_read(rbdr->rbdr_status) & NICVF_RBDR_COUNT_MASK))); 378 379 next_tail = __atomic_fetch_add(&rbdr->next_tail, to_fill, 380 __ATOMIC_ACQUIRE); 381 ltail = next_tail; 382 for (i = 0; i < to_fill; i++) { 383 struct rbdr_entry_t *entry = desc + (ltail & qlen_mask); 384 385 entry->full_addr = nicvf_mbuff_virt2phy((uintptr_t)obj_p[i], 386 mbuf_phys_off); 387 ltail++; 388 } 389 390 while (__atomic_load_n(&rbdr->tail, __ATOMIC_RELAXED) != next_tail) 391 rte_pause(); 392 393 __atomic_store_n(&rbdr->tail, ltail, __ATOMIC_RELEASE); 394 nicvf_addr_write(door, to_fill); 395 return to_fill; 396 } 397 398 static inline int32_t __hot 399 nicvf_rx_pkts_to_process(struct nicvf_rxq *rxq, uint16_t nb_pkts, 400 int32_t available_space) 401 { 402 if (unlikely(available_space < nb_pkts)) 403 rxq->available_space = nicvf_addr_read(rxq->cq_status) 404 & NICVF_CQ_CQE_COUNT_MASK; 405 406 return RTE_MIN(nb_pkts, available_space); 407 } 408 409 static inline void __hot 410 nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2, 411 struct rte_mbuf *pkt) 412 { 413 if (likely(cqe_rx_w0.rss_alg)) { 414 pkt->hash.rss = cqe_rx_w2.rss_tag; 415 pkt->ol_flags |= PKT_RX_RSS_HASH; 416 } 417 } 418 419 uint16_t __hot 420 nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 421 { 422 uint32_t i, to_process; 423 struct cqe_rx_t *cqe_rx; 424 struct rte_mbuf *pkt; 425 cqe_rx_word0_t cqe_rx_w0; 426 cqe_rx_word1_t cqe_rx_w1; 427 cqe_rx_word2_t cqe_rx_w2; 428 cqe_rx_word3_t cqe_rx_w3; 429 struct nicvf_rxq *rxq = rx_queue; 430 union cq_entry_t *desc = rxq->desc; 431 const uint64_t cqe_mask = rxq->qlen_mask; 432 uint64_t rb0_ptr, mbuf_phys_off = rxq->mbuf_phys_off; 433 uint32_t cqe_head = rxq->head & cqe_mask; 434 int32_t available_space = rxq->available_space; 435 uint8_t port_id = rxq->port_id; 436 const uint8_t rbptr_offset = rxq->rbptr_offset; 437 438 to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space); 439 440 for (i = 0; i < to_process; i++) { 441 rte_prefetch_non_temporal(&desc[cqe_head + 2]); 442 cqe_rx = (struct cqe_rx_t *)&desc[cqe_head]; 443 NICVF_RX_ASSERT(((struct cq_entry_type_t *)cqe_rx)->cqe_type 444 == CQE_TYPE_RX); 445 446 NICVF_LOAD_PAIR(cqe_rx_w0.u64, cqe_rx_w1.u64, cqe_rx); 447 NICVF_LOAD_PAIR(cqe_rx_w2.u64, cqe_rx_w3.u64, &cqe_rx->word2); 448 rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset); 449 pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt 450 (rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off); 451 452 pkt->ol_flags = 0; 453 pkt->port = port_id; 454 pkt->data_len = cqe_rx_w3.rb0_sz; 455 pkt->data_off = RTE_PKTMBUF_HEADROOM + cqe_rx_w1.align_pad; 456 pkt->nb_segs = 1; 457 pkt->pkt_len = cqe_rx_w3.rb0_sz; 458 pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); 459 460 nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); 461 rte_mbuf_refcnt_set(pkt, 1); 462 rx_pkts[i] = pkt; 463 cqe_head = (cqe_head + 1) & cqe_mask; 464 nicvf_prefetch_store_keep(pkt); 465 } 466 467 if (likely(to_process)) { 468 rxq->available_space -= to_process; 469 rxq->head = cqe_head; 470 nicvf_addr_write(rxq->cq_door, to_process); 471 rxq->recv_buffers += to_process; 472 if (rxq->recv_buffers > rxq->rx_free_thresh) { 473 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, 474 rxq->rx_free_thresh); 475 NICVF_RX_ASSERT(rxq->recv_buffers >= 0); 476 } 477 } 478 479 return to_process; 480 } 481 482 static inline uint16_t __hot 483 nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, 484 uint64_t mbuf_phys_off, uint8_t port_id, 485 struct rte_mbuf **rx_pkt, uint8_t rbptr_offset) 486 { 487 struct rte_mbuf *pkt, *seg, *prev; 488 cqe_rx_word0_t cqe_rx_w0; 489 cqe_rx_word1_t cqe_rx_w1; 490 cqe_rx_word2_t cqe_rx_w2; 491 uint16_t *rb_sz, nb_segs, seg_idx; 492 uint64_t *rb_ptr; 493 494 NICVF_LOAD_PAIR(cqe_rx_w0.u64, cqe_rx_w1.u64, cqe_rx); 495 NICVF_RX_ASSERT(cqe_rx_w0.cqe_type == CQE_TYPE_RX); 496 cqe_rx_w2 = cqe_rx->word2; 497 rb_sz = &cqe_rx->word3.rb0_sz; 498 rb_ptr = (uint64_t *)cqe_rx + rbptr_offset; 499 nb_segs = cqe_rx_w0.rb_cnt; 500 pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt 501 (rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off); 502 503 pkt->ol_flags = 0; 504 pkt->port = port_id; 505 pkt->data_off = RTE_PKTMBUF_HEADROOM + cqe_rx_w1.align_pad; 506 pkt->nb_segs = nb_segs; 507 pkt->pkt_len = cqe_rx_w1.pkt_len; 508 pkt->data_len = rb_sz[nicvf_frag_num(0)]; 509 rte_mbuf_refcnt_set(pkt, 1); 510 pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0); 511 nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt); 512 513 *rx_pkt = pkt; 514 prev = pkt; 515 for (seg_idx = 1; seg_idx < nb_segs; seg_idx++) { 516 seg = (struct rte_mbuf *)nicvf_mbuff_phy2virt 517 (rb_ptr[seg_idx], mbuf_phys_off); 518 519 prev->next = seg; 520 seg->data_len = rb_sz[nicvf_frag_num(seg_idx)]; 521 seg->port = port_id; 522 seg->data_off = RTE_PKTMBUF_HEADROOM; 523 rte_mbuf_refcnt_set(seg, 1); 524 525 prev = seg; 526 } 527 prev->next = NULL; 528 return nb_segs; 529 } 530 531 uint16_t __hot 532 nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, 533 uint16_t nb_pkts) 534 { 535 union cq_entry_t *cq_entry; 536 struct cqe_rx_t *cqe_rx; 537 struct nicvf_rxq *rxq = rx_queue; 538 union cq_entry_t *desc = rxq->desc; 539 const uint64_t cqe_mask = rxq->qlen_mask; 540 uint64_t mbuf_phys_off = rxq->mbuf_phys_off; 541 uint32_t i, to_process, cqe_head, buffers_consumed = 0; 542 int32_t available_space = rxq->available_space; 543 uint16_t nb_segs; 544 const uint8_t port_id = rxq->port_id; 545 const uint8_t rbptr_offset = rxq->rbptr_offset; 546 547 cqe_head = rxq->head & cqe_mask; 548 to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space); 549 550 for (i = 0; i < to_process; i++) { 551 rte_prefetch_non_temporal(&desc[cqe_head + 2]); 552 cq_entry = &desc[cqe_head]; 553 cqe_rx = (struct cqe_rx_t *)cq_entry; 554 nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off, 555 port_id, rx_pkts + i, rbptr_offset); 556 buffers_consumed += nb_segs; 557 cqe_head = (cqe_head + 1) & cqe_mask; 558 nicvf_prefetch_store_keep(rx_pkts[i]); 559 } 560 561 if (likely(to_process)) { 562 rxq->available_space -= to_process; 563 rxq->head = cqe_head; 564 nicvf_addr_write(rxq->cq_door, to_process); 565 rxq->recv_buffers += buffers_consumed; 566 if (rxq->recv_buffers > rxq->rx_free_thresh) { 567 rxq->recv_buffers -= 568 nicvf_fill_rbdr(rxq, rxq->rx_free_thresh); 569 NICVF_RX_ASSERT(rxq->recv_buffers >= 0); 570 } 571 } 572 573 return to_process; 574 } 575 576 uint32_t 577 nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) 578 { 579 struct nicvf_rxq *rxq; 580 581 rxq = dev->data->rx_queues[queue_idx]; 582 return nicvf_addr_read(rxq->cq_status) & NICVF_CQ_CQE_COUNT_MASK; 583 } 584 585 uint32_t 586 nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx) 587 { 588 struct nicvf_rxq *rxq; 589 uint32_t to_process; 590 uint32_t rx_free; 591 592 rxq = dev->data->rx_queues[queue_idx]; 593 to_process = rxq->recv_buffers; 594 while (rxq->recv_buffers > 0) { 595 rx_free = RTE_MIN(rxq->recv_buffers, NICVF_MAX_RX_FREE_THRESH); 596 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rx_free); 597 } 598 599 assert(rxq->recv_buffers == 0); 600 return to_process; 601 } 602