1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_cycles.h> 12 #include <rte_memory.h> 13 #include <rte_branch_prediction.h> 14 #include <rte_mempool.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_ether.h> 18 #include <rte_ethdev_driver.h> 19 #include <rte_prefetch.h> 20 #include <rte_string_fns.h> 21 #include <rte_errno.h> 22 #include <rte_byteorder.h> 23 #include <rte_net.h> 24 #include <rte_ip.h> 25 #include <rte_udp.h> 26 #include <rte_tcp.h> 27 28 #include "virtio_logs.h" 29 #include "virtio_ethdev.h" 30 #include "virtio_pci.h" 31 #include "virtqueue.h" 32 #include "virtio_rxtx.h" 33 #include "virtio_rxtx_simple.h" 34 #include "virtio_ring.h" 35 36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 38 #else 39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 40 #endif 41 42 int 43 virtio_dev_rx_queue_done(void *rxq, uint16_t offset) 44 { 45 struct virtnet_rx *rxvq = rxq; 46 struct virtqueue *vq = rxvq->vq; 47 48 return VIRTQUEUE_NUSED(vq) >= offset; 49 } 50 51 void 52 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 53 { 54 vq->vq_free_cnt += num; 55 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 56 } 57 58 void 59 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 60 { 61 struct vring_desc *dp, *dp_tail; 62 struct vq_desc_extra *dxp; 63 uint16_t desc_idx_last = desc_idx; 64 65 dp = &vq->vq_split.ring.desc[desc_idx]; 66 dxp = &vq->vq_descx[desc_idx]; 67 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 68 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 69 while (dp->flags & VRING_DESC_F_NEXT) { 70 desc_idx_last = dp->next; 71 dp = &vq->vq_split.ring.desc[dp->next]; 72 } 73 } 74 dxp->ndescs = 0; 75 76 /* 77 * We must append the existing free chain, if any, to the end of 78 * newly freed chain. If the virtqueue was completely used, then 79 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 80 */ 81 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 82 vq->vq_desc_head_idx = desc_idx; 83 } else { 84 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 85 dp_tail->next = desc_idx; 86 } 87 88 vq->vq_desc_tail_idx = desc_idx_last; 89 dp->next = VQ_RING_DESC_CHAIN_END; 90 } 91 92 static void 93 vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id) 94 { 95 struct vq_desc_extra *dxp; 96 97 dxp = &vq->vq_descx[id]; 98 vq->vq_free_cnt += dxp->ndescs; 99 100 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) 101 vq->vq_desc_head_idx = id; 102 else 103 vq->vq_descx[vq->vq_desc_tail_idx].next = id; 104 105 vq->vq_desc_tail_idx = id; 106 dxp->next = VQ_RING_DESC_CHAIN_END; 107 } 108 109 void 110 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) 111 { 112 uint32_t s = mbuf->pkt_len; 113 struct rte_ether_addr *ea; 114 115 stats->bytes += s; 116 117 if (s == 64) { 118 stats->size_bins[1]++; 119 } else if (s > 64 && s < 1024) { 120 uint32_t bin; 121 122 /* count zeros, and offset into correct bin */ 123 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 124 stats->size_bins[bin]++; 125 } else { 126 if (s < 64) 127 stats->size_bins[0]++; 128 else if (s < 1519) 129 stats->size_bins[6]++; 130 else 131 stats->size_bins[7]++; 132 } 133 134 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 135 if (rte_is_multicast_ether_addr(ea)) { 136 if (rte_is_broadcast_ether_addr(ea)) 137 stats->broadcast++; 138 else 139 stats->multicast++; 140 } 141 } 142 143 static inline void 144 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 145 { 146 VIRTIO_DUMP_PACKET(m, m->data_len); 147 148 virtio_update_packet_stats(&rxvq->stats, m); 149 } 150 151 static uint16_t 152 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 153 struct rte_mbuf **rx_pkts, 154 uint32_t *len, 155 uint16_t num) 156 { 157 struct rte_mbuf *cookie; 158 uint16_t used_idx; 159 uint16_t id; 160 struct vring_packed_desc *desc; 161 uint16_t i; 162 163 desc = vq->vq_packed.ring.desc; 164 165 for (i = 0; i < num; i++) { 166 used_idx = vq->vq_used_cons_idx; 167 /* desc_is_used has a load-acquire or rte_cio_rmb inside 168 * and wait for used desc in virtqueue. 169 */ 170 if (!desc_is_used(&desc[used_idx], vq)) 171 return i; 172 len[i] = desc[used_idx].len; 173 id = desc[used_idx].id; 174 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 175 if (unlikely(cookie == NULL)) { 176 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 177 vq->vq_used_cons_idx); 178 break; 179 } 180 rte_prefetch0(cookie); 181 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 182 rx_pkts[i] = cookie; 183 184 vq->vq_free_cnt++; 185 vq->vq_used_cons_idx++; 186 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 187 vq->vq_used_cons_idx -= vq->vq_nentries; 188 vq->vq_packed.used_wrap_counter ^= 1; 189 } 190 } 191 192 return i; 193 } 194 195 static uint16_t 196 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 197 uint32_t *len, uint16_t num) 198 { 199 struct vring_used_elem *uep; 200 struct rte_mbuf *cookie; 201 uint16_t used_idx, desc_idx; 202 uint16_t i; 203 204 /* Caller does the check */ 205 for (i = 0; i < num ; i++) { 206 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 207 uep = &vq->vq_split.ring.used->ring[used_idx]; 208 desc_idx = (uint16_t) uep->id; 209 len[i] = uep->len; 210 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 211 212 if (unlikely(cookie == NULL)) { 213 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 214 vq->vq_used_cons_idx); 215 break; 216 } 217 218 rte_prefetch0(cookie); 219 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 220 rx_pkts[i] = cookie; 221 vq->vq_used_cons_idx++; 222 vq_ring_free_chain(vq, desc_idx); 223 vq->vq_descx[desc_idx].cookie = NULL; 224 } 225 226 return i; 227 } 228 229 static uint16_t 230 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 231 struct rte_mbuf **rx_pkts, 232 uint32_t *len, 233 uint16_t num) 234 { 235 struct vring_used_elem *uep; 236 struct rte_mbuf *cookie; 237 uint16_t used_idx = 0; 238 uint16_t i; 239 240 if (unlikely(num == 0)) 241 return 0; 242 243 for (i = 0; i < num; i++) { 244 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 245 /* Desc idx same as used idx */ 246 uep = &vq->vq_split.ring.used->ring[used_idx]; 247 len[i] = uep->len; 248 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 249 250 if (unlikely(cookie == NULL)) { 251 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 252 vq->vq_used_cons_idx); 253 break; 254 } 255 256 rte_prefetch0(cookie); 257 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 258 rx_pkts[i] = cookie; 259 vq->vq_used_cons_idx++; 260 vq->vq_descx[used_idx].cookie = NULL; 261 } 262 263 vq_ring_free_inorder(vq, used_idx, i); 264 return i; 265 } 266 267 #ifndef DEFAULT_TX_FREE_THRESH 268 #define DEFAULT_TX_FREE_THRESH 32 269 #endif 270 271 static void 272 virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num) 273 { 274 uint16_t used_idx, id, curr_id, free_cnt = 0; 275 uint16_t size = vq->vq_nentries; 276 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; 277 struct vq_desc_extra *dxp; 278 279 used_idx = vq->vq_used_cons_idx; 280 /* desc_is_used has a load-acquire or rte_cio_rmb inside 281 * and wait for used desc in virtqueue. 282 */ 283 while (num > 0 && desc_is_used(&desc[used_idx], vq)) { 284 id = desc[used_idx].id; 285 do { 286 curr_id = used_idx; 287 dxp = &vq->vq_descx[used_idx]; 288 used_idx += dxp->ndescs; 289 free_cnt += dxp->ndescs; 290 num -= dxp->ndescs; 291 if (used_idx >= size) { 292 used_idx -= size; 293 vq->vq_packed.used_wrap_counter ^= 1; 294 } 295 if (dxp->cookie != NULL) { 296 rte_pktmbuf_free(dxp->cookie); 297 dxp->cookie = NULL; 298 } 299 } while (curr_id != id); 300 } 301 vq->vq_used_cons_idx = used_idx; 302 vq->vq_free_cnt += free_cnt; 303 } 304 305 static void 306 virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num) 307 { 308 uint16_t used_idx, id; 309 uint16_t size = vq->vq_nentries; 310 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; 311 struct vq_desc_extra *dxp; 312 313 used_idx = vq->vq_used_cons_idx; 314 /* desc_is_used has a load-acquire or rte_cio_rmb inside 315 * and wait for used desc in virtqueue. 316 */ 317 while (num-- && desc_is_used(&desc[used_idx], vq)) { 318 id = desc[used_idx].id; 319 dxp = &vq->vq_descx[id]; 320 vq->vq_used_cons_idx += dxp->ndescs; 321 if (vq->vq_used_cons_idx >= size) { 322 vq->vq_used_cons_idx -= size; 323 vq->vq_packed.used_wrap_counter ^= 1; 324 } 325 vq_ring_free_id_packed(vq, id); 326 if (dxp->cookie != NULL) { 327 rte_pktmbuf_free(dxp->cookie); 328 dxp->cookie = NULL; 329 } 330 used_idx = vq->vq_used_cons_idx; 331 } 332 } 333 334 /* Cleanup from completed transmits. */ 335 static inline void 336 virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order) 337 { 338 if (in_order) 339 virtio_xmit_cleanup_inorder_packed(vq, num); 340 else 341 virtio_xmit_cleanup_normal_packed(vq, num); 342 } 343 344 static void 345 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) 346 { 347 uint16_t i, used_idx, desc_idx; 348 for (i = 0; i < num; i++) { 349 struct vring_used_elem *uep; 350 struct vq_desc_extra *dxp; 351 352 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 353 uep = &vq->vq_split.ring.used->ring[used_idx]; 354 355 desc_idx = (uint16_t) uep->id; 356 dxp = &vq->vq_descx[desc_idx]; 357 vq->vq_used_cons_idx++; 358 vq_ring_free_chain(vq, desc_idx); 359 360 if (dxp->cookie != NULL) { 361 rte_pktmbuf_free(dxp->cookie); 362 dxp->cookie = NULL; 363 } 364 } 365 } 366 367 /* Cleanup from completed inorder transmits. */ 368 static __rte_always_inline void 369 virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num) 370 { 371 uint16_t i, idx = vq->vq_used_cons_idx; 372 int16_t free_cnt = 0; 373 struct vq_desc_extra *dxp = NULL; 374 375 if (unlikely(num == 0)) 376 return; 377 378 for (i = 0; i < num; i++) { 379 dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)]; 380 free_cnt += dxp->ndescs; 381 if (dxp->cookie != NULL) { 382 rte_pktmbuf_free(dxp->cookie); 383 dxp->cookie = NULL; 384 } 385 } 386 387 vq->vq_free_cnt += free_cnt; 388 vq->vq_used_cons_idx = idx; 389 } 390 391 static inline int 392 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 393 struct rte_mbuf **cookies, 394 uint16_t num) 395 { 396 struct vq_desc_extra *dxp; 397 struct virtio_hw *hw = vq->hw; 398 struct vring_desc *start_dp; 399 uint16_t head_idx, idx, i = 0; 400 401 if (unlikely(vq->vq_free_cnt == 0)) 402 return -ENOSPC; 403 if (unlikely(vq->vq_free_cnt < num)) 404 return -EMSGSIZE; 405 406 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 407 start_dp = vq->vq_split.ring.desc; 408 409 while (i < num) { 410 idx = head_idx & (vq->vq_nentries - 1); 411 dxp = &vq->vq_descx[idx]; 412 dxp->cookie = (void *)cookies[i]; 413 dxp->ndescs = 1; 414 415 start_dp[idx].addr = 416 VIRTIO_MBUF_ADDR(cookies[i], vq) + 417 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 418 start_dp[idx].len = 419 cookies[i]->buf_len - 420 RTE_PKTMBUF_HEADROOM + 421 hw->vtnet_hdr_size; 422 start_dp[idx].flags = VRING_DESC_F_WRITE; 423 424 vq_update_avail_ring(vq, idx); 425 head_idx++; 426 i++; 427 } 428 429 vq->vq_desc_head_idx += num; 430 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 431 return 0; 432 } 433 434 static inline int 435 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 436 uint16_t num) 437 { 438 struct vq_desc_extra *dxp; 439 struct virtio_hw *hw = vq->hw; 440 struct vring_desc *start_dp = vq->vq_split.ring.desc; 441 uint16_t idx, i; 442 443 if (unlikely(vq->vq_free_cnt == 0)) 444 return -ENOSPC; 445 if (unlikely(vq->vq_free_cnt < num)) 446 return -EMSGSIZE; 447 448 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 449 return -EFAULT; 450 451 for (i = 0; i < num; i++) { 452 idx = vq->vq_desc_head_idx; 453 dxp = &vq->vq_descx[idx]; 454 dxp->cookie = (void *)cookie[i]; 455 dxp->ndescs = 1; 456 457 start_dp[idx].addr = 458 VIRTIO_MBUF_ADDR(cookie[i], vq) + 459 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 460 start_dp[idx].len = 461 cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + 462 hw->vtnet_hdr_size; 463 start_dp[idx].flags = VRING_DESC_F_WRITE; 464 vq->vq_desc_head_idx = start_dp[idx].next; 465 vq_update_avail_ring(vq, idx); 466 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 467 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 468 break; 469 } 470 } 471 472 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 473 474 return 0; 475 } 476 477 static inline int 478 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 479 struct rte_mbuf **cookie, uint16_t num) 480 { 481 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 482 uint16_t flags = vq->vq_packed.cached_flags; 483 struct virtio_hw *hw = vq->hw; 484 struct vq_desc_extra *dxp; 485 uint16_t idx; 486 int i; 487 488 if (unlikely(vq->vq_free_cnt == 0)) 489 return -ENOSPC; 490 if (unlikely(vq->vq_free_cnt < num)) 491 return -EMSGSIZE; 492 493 for (i = 0; i < num; i++) { 494 idx = vq->vq_avail_idx; 495 dxp = &vq->vq_descx[idx]; 496 dxp->cookie = (void *)cookie[i]; 497 dxp->ndescs = 1; 498 499 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + 500 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 501 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM 502 + hw->vtnet_hdr_size; 503 504 vq->vq_desc_head_idx = dxp->next; 505 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 506 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 507 508 virtqueue_store_flags_packed(&start_dp[idx], flags, 509 hw->weak_barriers); 510 511 if (++vq->vq_avail_idx >= vq->vq_nentries) { 512 vq->vq_avail_idx -= vq->vq_nentries; 513 vq->vq_packed.cached_flags ^= 514 VRING_PACKED_DESC_F_AVAIL_USED; 515 flags = vq->vq_packed.cached_flags; 516 } 517 } 518 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 519 return 0; 520 } 521 522 /* When doing TSO, the IP length is not included in the pseudo header 523 * checksum of the packet given to the PMD, but for virtio it is 524 * expected. 525 */ 526 static void 527 virtio_tso_fix_cksum(struct rte_mbuf *m) 528 { 529 /* common case: header is not fragmented */ 530 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 531 m->l4_len)) { 532 struct rte_ipv4_hdr *iph; 533 struct rte_ipv6_hdr *ip6h; 534 struct rte_tcp_hdr *th; 535 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; 536 uint32_t tmp; 537 538 iph = rte_pktmbuf_mtod_offset(m, 539 struct rte_ipv4_hdr *, m->l2_len); 540 th = RTE_PTR_ADD(iph, m->l3_len); 541 if ((iph->version_ihl >> 4) == 4) { 542 iph->hdr_checksum = 0; 543 iph->hdr_checksum = rte_ipv4_cksum(iph); 544 ip_len = iph->total_length; 545 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - 546 m->l3_len); 547 } else { 548 ip6h = (struct rte_ipv6_hdr *)iph; 549 ip_paylen = ip6h->payload_len; 550 } 551 552 /* calculate the new phdr checksum not including ip_paylen */ 553 prev_cksum = th->cksum; 554 tmp = prev_cksum; 555 tmp += ip_paylen; 556 tmp = (tmp & 0xffff) + (tmp >> 16); 557 new_cksum = tmp; 558 559 /* replace it in the packet */ 560 th->cksum = new_cksum; 561 } 562 } 563 564 565 /* avoid write operation when necessary, to lessen cache issues */ 566 #define ASSIGN_UNLESS_EQUAL(var, val) do { \ 567 if ((var) != (val)) \ 568 (var) = (val); \ 569 } while (0) 570 571 #define virtqueue_clear_net_hdr(_hdr) do { \ 572 ASSIGN_UNLESS_EQUAL((_hdr)->csum_start, 0); \ 573 ASSIGN_UNLESS_EQUAL((_hdr)->csum_offset, 0); \ 574 ASSIGN_UNLESS_EQUAL((_hdr)->flags, 0); \ 575 ASSIGN_UNLESS_EQUAL((_hdr)->gso_type, 0); \ 576 ASSIGN_UNLESS_EQUAL((_hdr)->gso_size, 0); \ 577 ASSIGN_UNLESS_EQUAL((_hdr)->hdr_len, 0); \ 578 } while (0) 579 580 static inline void 581 virtqueue_xmit_offload(struct virtio_net_hdr *hdr, 582 struct rte_mbuf *cookie, 583 bool offload) 584 { 585 if (offload) { 586 if (cookie->ol_flags & PKT_TX_TCP_SEG) 587 cookie->ol_flags |= PKT_TX_TCP_CKSUM; 588 589 switch (cookie->ol_flags & PKT_TX_L4_MASK) { 590 case PKT_TX_UDP_CKSUM: 591 hdr->csum_start = cookie->l2_len + cookie->l3_len; 592 hdr->csum_offset = offsetof(struct rte_udp_hdr, 593 dgram_cksum); 594 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 595 break; 596 597 case PKT_TX_TCP_CKSUM: 598 hdr->csum_start = cookie->l2_len + cookie->l3_len; 599 hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum); 600 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 601 break; 602 603 default: 604 ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); 605 ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); 606 ASSIGN_UNLESS_EQUAL(hdr->flags, 0); 607 break; 608 } 609 610 /* TCP Segmentation Offload */ 611 if (cookie->ol_flags & PKT_TX_TCP_SEG) { 612 hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ? 613 VIRTIO_NET_HDR_GSO_TCPV6 : 614 VIRTIO_NET_HDR_GSO_TCPV4; 615 hdr->gso_size = cookie->tso_segsz; 616 hdr->hdr_len = 617 cookie->l2_len + 618 cookie->l3_len + 619 cookie->l4_len; 620 } else { 621 ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); 622 ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); 623 ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); 624 } 625 } 626 } 627 628 static inline void 629 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 630 struct rte_mbuf **cookies, 631 uint16_t num) 632 { 633 struct vq_desc_extra *dxp; 634 struct virtqueue *vq = txvq->vq; 635 struct vring_desc *start_dp; 636 struct virtio_net_hdr *hdr; 637 uint16_t idx; 638 int16_t head_size = vq->hw->vtnet_hdr_size; 639 uint16_t i = 0; 640 641 idx = vq->vq_desc_head_idx; 642 start_dp = vq->vq_split.ring.desc; 643 644 while (i < num) { 645 idx = idx & (vq->vq_nentries - 1); 646 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 647 dxp->cookie = (void *)cookies[i]; 648 dxp->ndescs = 1; 649 virtio_update_packet_stats(&txvq->stats, cookies[i]); 650 651 hdr = rte_pktmbuf_mtod_offset(cookies[i], 652 struct virtio_net_hdr *, -head_size); 653 654 /* if offload disabled, hdr is not zeroed yet, do it now */ 655 if (!vq->hw->has_tx_offload) 656 virtqueue_clear_net_hdr(hdr); 657 else 658 virtqueue_xmit_offload(hdr, cookies[i], true); 659 660 start_dp[idx].addr = 661 VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size; 662 start_dp[idx].len = cookies[i]->data_len + head_size; 663 start_dp[idx].flags = 0; 664 665 666 vq_update_avail_ring(vq, idx); 667 668 idx++; 669 i++; 670 }; 671 672 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 673 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 674 } 675 676 static inline void 677 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 678 struct rte_mbuf *cookie, 679 int in_order) 680 { 681 struct virtqueue *vq = txvq->vq; 682 struct vring_packed_desc *dp; 683 struct vq_desc_extra *dxp; 684 uint16_t idx, id, flags; 685 int16_t head_size = vq->hw->vtnet_hdr_size; 686 struct virtio_net_hdr *hdr; 687 688 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 689 idx = vq->vq_avail_idx; 690 dp = &vq->vq_packed.ring.desc[idx]; 691 692 dxp = &vq->vq_descx[id]; 693 dxp->ndescs = 1; 694 dxp->cookie = cookie; 695 696 flags = vq->vq_packed.cached_flags; 697 698 /* prepend cannot fail, checked by caller */ 699 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 700 -head_size); 701 702 /* if offload disabled, hdr is not zeroed yet, do it now */ 703 if (!vq->hw->has_tx_offload) 704 virtqueue_clear_net_hdr(hdr); 705 else 706 virtqueue_xmit_offload(hdr, cookie, true); 707 708 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; 709 dp->len = cookie->data_len + head_size; 710 dp->id = id; 711 712 if (++vq->vq_avail_idx >= vq->vq_nentries) { 713 vq->vq_avail_idx -= vq->vq_nentries; 714 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 715 } 716 717 vq->vq_free_cnt--; 718 719 if (!in_order) { 720 vq->vq_desc_head_idx = dxp->next; 721 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 722 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 723 } 724 725 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 726 } 727 728 static inline void 729 virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 730 uint16_t needed, int can_push, int in_order) 731 { 732 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 733 struct vq_desc_extra *dxp; 734 struct virtqueue *vq = txvq->vq; 735 struct vring_packed_desc *start_dp, *head_dp; 736 uint16_t idx, id, head_idx, head_flags; 737 int16_t head_size = vq->hw->vtnet_hdr_size; 738 struct virtio_net_hdr *hdr; 739 uint16_t prev; 740 bool prepend_header = false; 741 742 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 743 744 dxp = &vq->vq_descx[id]; 745 dxp->ndescs = needed; 746 dxp->cookie = cookie; 747 748 head_idx = vq->vq_avail_idx; 749 idx = head_idx; 750 prev = head_idx; 751 start_dp = vq->vq_packed.ring.desc; 752 753 head_dp = &vq->vq_packed.ring.desc[idx]; 754 head_flags = cookie->next ? VRING_DESC_F_NEXT : 0; 755 head_flags |= vq->vq_packed.cached_flags; 756 757 if (can_push) { 758 /* prepend cannot fail, checked by caller */ 759 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 760 -head_size); 761 prepend_header = true; 762 763 /* if offload disabled, it is not zeroed below, do it now */ 764 if (!vq->hw->has_tx_offload) 765 virtqueue_clear_net_hdr(hdr); 766 } else { 767 /* setup first tx ring slot to point to header 768 * stored in reserved region. 769 */ 770 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 771 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 772 start_dp[idx].len = vq->hw->vtnet_hdr_size; 773 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 774 idx++; 775 if (idx >= vq->vq_nentries) { 776 idx -= vq->vq_nentries; 777 vq->vq_packed.cached_flags ^= 778 VRING_PACKED_DESC_F_AVAIL_USED; 779 } 780 } 781 782 virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload); 783 784 do { 785 uint16_t flags; 786 787 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 788 start_dp[idx].len = cookie->data_len; 789 if (prepend_header) { 790 start_dp[idx].addr -= head_size; 791 start_dp[idx].len += head_size; 792 prepend_header = false; 793 } 794 795 if (likely(idx != head_idx)) { 796 flags = cookie->next ? VRING_DESC_F_NEXT : 0; 797 flags |= vq->vq_packed.cached_flags; 798 start_dp[idx].flags = flags; 799 } 800 prev = idx; 801 idx++; 802 if (idx >= vq->vq_nentries) { 803 idx -= vq->vq_nentries; 804 vq->vq_packed.cached_flags ^= 805 VRING_PACKED_DESC_F_AVAIL_USED; 806 } 807 } while ((cookie = cookie->next) != NULL); 808 809 start_dp[prev].id = id; 810 811 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 812 vq->vq_avail_idx = idx; 813 814 if (!in_order) { 815 vq->vq_desc_head_idx = dxp->next; 816 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 817 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 818 } 819 820 virtqueue_store_flags_packed(head_dp, head_flags, 821 vq->hw->weak_barriers); 822 } 823 824 static inline void 825 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 826 uint16_t needed, int use_indirect, int can_push, 827 int in_order) 828 { 829 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 830 struct vq_desc_extra *dxp; 831 struct virtqueue *vq = txvq->vq; 832 struct vring_desc *start_dp; 833 uint16_t seg_num = cookie->nb_segs; 834 uint16_t head_idx, idx; 835 int16_t head_size = vq->hw->vtnet_hdr_size; 836 bool prepend_header = false; 837 struct virtio_net_hdr *hdr; 838 839 head_idx = vq->vq_desc_head_idx; 840 idx = head_idx; 841 if (in_order) 842 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 843 else 844 dxp = &vq->vq_descx[idx]; 845 dxp->cookie = (void *)cookie; 846 dxp->ndescs = needed; 847 848 start_dp = vq->vq_split.ring.desc; 849 850 if (can_push) { 851 /* prepend cannot fail, checked by caller */ 852 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 853 -head_size); 854 prepend_header = true; 855 856 /* if offload disabled, it is not zeroed below, do it now */ 857 if (!vq->hw->has_tx_offload) 858 virtqueue_clear_net_hdr(hdr); 859 } else if (use_indirect) { 860 /* setup tx ring slot to point to indirect 861 * descriptor list stored in reserved region. 862 * 863 * the first slot in indirect ring is already preset 864 * to point to the header in reserved region 865 */ 866 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 867 RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 868 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 869 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 870 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 871 872 /* loop below will fill in rest of the indirect elements */ 873 start_dp = txr[idx].tx_indir; 874 idx = 1; 875 } else { 876 /* setup first tx ring slot to point to header 877 * stored in reserved region. 878 */ 879 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 880 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 881 start_dp[idx].len = vq->hw->vtnet_hdr_size; 882 start_dp[idx].flags = VRING_DESC_F_NEXT; 883 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 884 885 idx = start_dp[idx].next; 886 } 887 888 virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload); 889 890 do { 891 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 892 start_dp[idx].len = cookie->data_len; 893 if (prepend_header) { 894 start_dp[idx].addr -= head_size; 895 start_dp[idx].len += head_size; 896 prepend_header = false; 897 } 898 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 899 idx = start_dp[idx].next; 900 } while ((cookie = cookie->next) != NULL); 901 902 if (use_indirect) 903 idx = vq->vq_split.ring.desc[head_idx].next; 904 905 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 906 907 vq->vq_desc_head_idx = idx; 908 vq_update_avail_ring(vq, head_idx); 909 910 if (!in_order) { 911 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 912 vq->vq_desc_tail_idx = idx; 913 } 914 } 915 916 void 917 virtio_dev_cq_start(struct rte_eth_dev *dev) 918 { 919 struct virtio_hw *hw = dev->data->dev_private; 920 921 if (hw->cvq && hw->cvq->vq) { 922 rte_spinlock_init(&hw->cvq->lock); 923 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq->vq); 924 } 925 } 926 927 int 928 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 929 uint16_t queue_idx, 930 uint16_t nb_desc, 931 unsigned int socket_id __rte_unused, 932 const struct rte_eth_rxconf *rx_conf, 933 struct rte_mempool *mp) 934 { 935 uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 936 struct virtio_hw *hw = dev->data->dev_private; 937 struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; 938 struct virtnet_rx *rxvq; 939 940 PMD_INIT_FUNC_TRACE(); 941 942 if (rx_conf->rx_deferred_start) { 943 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 944 return -EINVAL; 945 } 946 947 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 948 nb_desc = vq->vq_nentries; 949 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 950 951 rxvq = &vq->rxq; 952 rxvq->queue_id = queue_idx; 953 rxvq->mpool = mp; 954 dev->data->rx_queues[queue_idx] = rxvq; 955 956 return 0; 957 } 958 959 int 960 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 961 { 962 uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 963 struct virtio_hw *hw = dev->data->dev_private; 964 struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; 965 struct virtnet_rx *rxvq = &vq->rxq; 966 struct rte_mbuf *m; 967 uint16_t desc_idx; 968 int error, nbufs, i; 969 970 PMD_INIT_FUNC_TRACE(); 971 972 /* Allocate blank mbufs for the each rx descriptor */ 973 nbufs = 0; 974 975 if (hw->use_simple_rx) { 976 for (desc_idx = 0; desc_idx < vq->vq_nentries; 977 desc_idx++) { 978 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 979 vq->vq_split.ring.desc[desc_idx].flags = 980 VRING_DESC_F_WRITE; 981 } 982 983 virtio_rxq_vec_setup(rxvq); 984 } 985 986 memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf)); 987 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; 988 desc_idx++) { 989 vq->sw_ring[vq->vq_nentries + desc_idx] = 990 &rxvq->fake_mbuf; 991 } 992 993 if (hw->use_simple_rx) { 994 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 995 virtio_rxq_rearm_vec(rxvq); 996 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 997 } 998 } else if (hw->use_inorder_rx) { 999 if ((!virtqueue_full(vq))) { 1000 uint16_t free_cnt = vq->vq_free_cnt; 1001 struct rte_mbuf *pkts[free_cnt]; 1002 1003 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 1004 free_cnt)) { 1005 error = virtqueue_enqueue_refill_inorder(vq, 1006 pkts, 1007 free_cnt); 1008 if (unlikely(error)) { 1009 for (i = 0; i < free_cnt; i++) 1010 rte_pktmbuf_free(pkts[i]); 1011 } 1012 } 1013 1014 nbufs += free_cnt; 1015 vq_update_avail_idx(vq); 1016 } 1017 } else { 1018 while (!virtqueue_full(vq)) { 1019 m = rte_mbuf_raw_alloc(rxvq->mpool); 1020 if (m == NULL) 1021 break; 1022 1023 /* Enqueue allocated buffers */ 1024 if (vtpci_packed_queue(vq->hw)) 1025 error = virtqueue_enqueue_recv_refill_packed(vq, 1026 &m, 1); 1027 else 1028 error = virtqueue_enqueue_recv_refill(vq, 1029 &m, 1); 1030 if (error) { 1031 rte_pktmbuf_free(m); 1032 break; 1033 } 1034 nbufs++; 1035 } 1036 1037 if (!vtpci_packed_queue(vq->hw)) 1038 vq_update_avail_idx(vq); 1039 } 1040 1041 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); 1042 1043 VIRTQUEUE_DUMP(vq); 1044 1045 return 0; 1046 } 1047 1048 /* 1049 * struct rte_eth_dev *dev: Used to update dev 1050 * uint16_t nb_desc: Defaults to values read from config space 1051 * unsigned int socket_id: Used to allocate memzone 1052 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 1053 * uint16_t queue_idx: Just used as an index in dev txq list 1054 */ 1055 int 1056 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 1057 uint16_t queue_idx, 1058 uint16_t nb_desc, 1059 unsigned int socket_id __rte_unused, 1060 const struct rte_eth_txconf *tx_conf) 1061 { 1062 uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 1063 struct virtio_hw *hw = dev->data->dev_private; 1064 struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; 1065 struct virtnet_tx *txvq; 1066 uint16_t tx_free_thresh; 1067 1068 PMD_INIT_FUNC_TRACE(); 1069 1070 if (tx_conf->tx_deferred_start) { 1071 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 1072 return -EINVAL; 1073 } 1074 1075 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 1076 nb_desc = vq->vq_nentries; 1077 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 1078 1079 txvq = &vq->txq; 1080 txvq->queue_id = queue_idx; 1081 1082 tx_free_thresh = tx_conf->tx_free_thresh; 1083 if (tx_free_thresh == 0) 1084 tx_free_thresh = 1085 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 1086 1087 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 1088 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the " 1089 "number of TX entries minus 3 (%u)." 1090 " (tx_free_thresh=%u port=%u queue=%u)\n", 1091 vq->vq_nentries - 3, 1092 tx_free_thresh, dev->data->port_id, queue_idx); 1093 return -EINVAL; 1094 } 1095 1096 vq->vq_free_thresh = tx_free_thresh; 1097 1098 dev->data->tx_queues[queue_idx] = txvq; 1099 return 0; 1100 } 1101 1102 int 1103 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 1104 uint16_t queue_idx) 1105 { 1106 uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 1107 struct virtio_hw *hw = dev->data->dev_private; 1108 struct virtqueue *vq = hw->vqs[vtpci_queue_idx]; 1109 1110 PMD_INIT_FUNC_TRACE(); 1111 1112 if (!vtpci_packed_queue(hw)) { 1113 if (hw->use_inorder_tx) 1114 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 1115 } 1116 1117 VIRTQUEUE_DUMP(vq); 1118 1119 return 0; 1120 } 1121 1122 static inline void 1123 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 1124 { 1125 int error; 1126 /* 1127 * Requeue the discarded mbuf. This should always be 1128 * successful since it was just dequeued. 1129 */ 1130 if (vtpci_packed_queue(vq->hw)) 1131 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 1132 else 1133 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 1134 1135 if (unlikely(error)) { 1136 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); 1137 rte_pktmbuf_free(m); 1138 } 1139 } 1140 1141 static inline void 1142 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 1143 { 1144 int error; 1145 1146 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 1147 if (unlikely(error)) { 1148 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); 1149 rte_pktmbuf_free(m); 1150 } 1151 } 1152 1153 /* Optionally fill offload information in structure */ 1154 static inline int 1155 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 1156 { 1157 struct rte_net_hdr_lens hdr_lens; 1158 uint32_t hdrlen, ptype; 1159 int l4_supported = 0; 1160 1161 /* nothing to do */ 1162 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 1163 return 0; 1164 1165 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; 1166 1167 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 1168 m->packet_type = ptype; 1169 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 1170 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 1171 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 1172 l4_supported = 1; 1173 1174 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 1175 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 1176 if (hdr->csum_start <= hdrlen && l4_supported) { 1177 m->ol_flags |= PKT_RX_L4_CKSUM_NONE; 1178 } else { 1179 /* Unknown proto or tunnel, do sw cksum. We can assume 1180 * the cksum field is in the first segment since the 1181 * buffers we provided to the host are large enough. 1182 * In case of SCTP, this will be wrong since it's a CRC 1183 * but there's nothing we can do. 1184 */ 1185 uint16_t csum = 0, off; 1186 1187 rte_raw_cksum_mbuf(m, hdr->csum_start, 1188 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 1189 &csum); 1190 if (likely(csum != 0xffff)) 1191 csum = ~csum; 1192 off = hdr->csum_offset + hdr->csum_start; 1193 if (rte_pktmbuf_data_len(m) >= off + 1) 1194 *rte_pktmbuf_mtod_offset(m, uint16_t *, 1195 off) = csum; 1196 } 1197 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 1198 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 1199 } 1200 1201 /* GSO request, save required information in mbuf */ 1202 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 1203 /* Check unsupported modes */ 1204 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 1205 (hdr->gso_size == 0)) { 1206 return -EINVAL; 1207 } 1208 1209 /* Update mss lengthes in mbuf */ 1210 m->tso_segsz = hdr->gso_size; 1211 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 1212 case VIRTIO_NET_HDR_GSO_TCPV4: 1213 case VIRTIO_NET_HDR_GSO_TCPV6: 1214 m->ol_flags |= PKT_RX_LRO | \ 1215 PKT_RX_L4_CKSUM_NONE; 1216 break; 1217 default: 1218 return -EINVAL; 1219 } 1220 } 1221 1222 return 0; 1223 } 1224 1225 #define VIRTIO_MBUF_BURST_SZ 64 1226 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 1227 uint16_t 1228 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1229 { 1230 struct virtnet_rx *rxvq = rx_queue; 1231 struct virtqueue *vq = rxvq->vq; 1232 struct virtio_hw *hw = vq->hw; 1233 struct rte_mbuf *rxm; 1234 uint16_t nb_used, num, nb_rx; 1235 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1236 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1237 int error; 1238 uint32_t i, nb_enqueued; 1239 uint32_t hdr_size; 1240 struct virtio_net_hdr *hdr; 1241 1242 nb_rx = 0; 1243 if (unlikely(hw->started == 0)) 1244 return nb_rx; 1245 1246 nb_used = VIRTQUEUE_NUSED(vq); 1247 1248 virtio_rmb(hw->weak_barriers); 1249 1250 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1251 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1252 num = VIRTIO_MBUF_BURST_SZ; 1253 if (likely(num > DESC_PER_CACHELINE)) 1254 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1255 1256 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1257 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 1258 1259 nb_enqueued = 0; 1260 hdr_size = hw->vtnet_hdr_size; 1261 1262 for (i = 0; i < num ; i++) { 1263 rxm = rcv_pkts[i]; 1264 1265 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1266 1267 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1268 PMD_RX_LOG(ERR, "Packet drop"); 1269 nb_enqueued++; 1270 virtio_discard_rxbuf(vq, rxm); 1271 rxvq->stats.errors++; 1272 continue; 1273 } 1274 1275 rxm->port = rxvq->port_id; 1276 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1277 rxm->ol_flags = 0; 1278 rxm->vlan_tci = 0; 1279 1280 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1281 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1282 1283 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1284 RTE_PKTMBUF_HEADROOM - hdr_size); 1285 1286 if (hw->vlan_strip) 1287 rte_vlan_strip(rxm); 1288 1289 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1290 virtio_discard_rxbuf(vq, rxm); 1291 rxvq->stats.errors++; 1292 continue; 1293 } 1294 1295 virtio_rx_stats_updated(rxvq, rxm); 1296 1297 rx_pkts[nb_rx++] = rxm; 1298 } 1299 1300 rxvq->stats.packets += nb_rx; 1301 1302 /* Allocate new mbuf for the used descriptor */ 1303 if (likely(!virtqueue_full(vq))) { 1304 uint16_t free_cnt = vq->vq_free_cnt; 1305 struct rte_mbuf *new_pkts[free_cnt]; 1306 1307 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1308 free_cnt) == 0)) { 1309 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1310 free_cnt); 1311 if (unlikely(error)) { 1312 for (i = 0; i < free_cnt; i++) 1313 rte_pktmbuf_free(new_pkts[i]); 1314 } 1315 nb_enqueued += free_cnt; 1316 } else { 1317 struct rte_eth_dev *dev = 1318 &rte_eth_devices[rxvq->port_id]; 1319 dev->data->rx_mbuf_alloc_failed += free_cnt; 1320 } 1321 } 1322 1323 if (likely(nb_enqueued)) { 1324 vq_update_avail_idx(vq); 1325 1326 if (unlikely(virtqueue_kick_prepare(vq))) { 1327 virtqueue_notify(vq); 1328 PMD_RX_LOG(DEBUG, "Notified"); 1329 } 1330 } 1331 1332 return nb_rx; 1333 } 1334 1335 uint16_t 1336 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1337 uint16_t nb_pkts) 1338 { 1339 struct virtnet_rx *rxvq = rx_queue; 1340 struct virtqueue *vq = rxvq->vq; 1341 struct virtio_hw *hw = vq->hw; 1342 struct rte_mbuf *rxm; 1343 uint16_t num, nb_rx; 1344 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1345 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1346 int error; 1347 uint32_t i, nb_enqueued; 1348 uint32_t hdr_size; 1349 struct virtio_net_hdr *hdr; 1350 1351 nb_rx = 0; 1352 if (unlikely(hw->started == 0)) 1353 return nb_rx; 1354 1355 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1356 if (likely(num > DESC_PER_CACHELINE)) 1357 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1358 1359 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1360 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1361 1362 nb_enqueued = 0; 1363 hdr_size = hw->vtnet_hdr_size; 1364 1365 for (i = 0; i < num; i++) { 1366 rxm = rcv_pkts[i]; 1367 1368 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1369 1370 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1371 PMD_RX_LOG(ERR, "Packet drop"); 1372 nb_enqueued++; 1373 virtio_discard_rxbuf(vq, rxm); 1374 rxvq->stats.errors++; 1375 continue; 1376 } 1377 1378 rxm->port = rxvq->port_id; 1379 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1380 rxm->ol_flags = 0; 1381 rxm->vlan_tci = 0; 1382 1383 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1384 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1385 1386 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1387 RTE_PKTMBUF_HEADROOM - hdr_size); 1388 1389 if (hw->vlan_strip) 1390 rte_vlan_strip(rxm); 1391 1392 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1393 virtio_discard_rxbuf(vq, rxm); 1394 rxvq->stats.errors++; 1395 continue; 1396 } 1397 1398 virtio_rx_stats_updated(rxvq, rxm); 1399 1400 rx_pkts[nb_rx++] = rxm; 1401 } 1402 1403 rxvq->stats.packets += nb_rx; 1404 1405 /* Allocate new mbuf for the used descriptor */ 1406 if (likely(!virtqueue_full(vq))) { 1407 uint16_t free_cnt = vq->vq_free_cnt; 1408 struct rte_mbuf *new_pkts[free_cnt]; 1409 1410 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1411 free_cnt) == 0)) { 1412 error = virtqueue_enqueue_recv_refill_packed(vq, 1413 new_pkts, free_cnt); 1414 if (unlikely(error)) { 1415 for (i = 0; i < free_cnt; i++) 1416 rte_pktmbuf_free(new_pkts[i]); 1417 } 1418 nb_enqueued += free_cnt; 1419 } else { 1420 struct rte_eth_dev *dev = 1421 &rte_eth_devices[rxvq->port_id]; 1422 dev->data->rx_mbuf_alloc_failed += free_cnt; 1423 } 1424 } 1425 1426 if (likely(nb_enqueued)) { 1427 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1428 virtqueue_notify(vq); 1429 PMD_RX_LOG(DEBUG, "Notified"); 1430 } 1431 } 1432 1433 return nb_rx; 1434 } 1435 1436 1437 uint16_t 1438 virtio_recv_pkts_inorder(void *rx_queue, 1439 struct rte_mbuf **rx_pkts, 1440 uint16_t nb_pkts) 1441 { 1442 struct virtnet_rx *rxvq = rx_queue; 1443 struct virtqueue *vq = rxvq->vq; 1444 struct virtio_hw *hw = vq->hw; 1445 struct rte_mbuf *rxm; 1446 struct rte_mbuf *prev = NULL; 1447 uint16_t nb_used, num, nb_rx; 1448 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1449 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1450 int error; 1451 uint32_t nb_enqueued; 1452 uint32_t seg_num; 1453 uint32_t seg_res; 1454 uint32_t hdr_size; 1455 int32_t i; 1456 1457 nb_rx = 0; 1458 if (unlikely(hw->started == 0)) 1459 return nb_rx; 1460 1461 nb_used = VIRTQUEUE_NUSED(vq); 1462 nb_used = RTE_MIN(nb_used, nb_pkts); 1463 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1464 1465 virtio_rmb(hw->weak_barriers); 1466 1467 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1468 1469 nb_enqueued = 0; 1470 seg_num = 1; 1471 seg_res = 0; 1472 hdr_size = hw->vtnet_hdr_size; 1473 1474 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1475 1476 for (i = 0; i < num; i++) { 1477 struct virtio_net_hdr_mrg_rxbuf *header; 1478 1479 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1480 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1481 1482 rxm = rcv_pkts[i]; 1483 1484 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1485 PMD_RX_LOG(ERR, "Packet drop"); 1486 nb_enqueued++; 1487 virtio_discard_rxbuf_inorder(vq, rxm); 1488 rxvq->stats.errors++; 1489 continue; 1490 } 1491 1492 header = (struct virtio_net_hdr_mrg_rxbuf *) 1493 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1494 - hdr_size); 1495 1496 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1497 seg_num = header->num_buffers; 1498 if (seg_num == 0) 1499 seg_num = 1; 1500 } else { 1501 seg_num = 1; 1502 } 1503 1504 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1505 rxm->nb_segs = seg_num; 1506 rxm->ol_flags = 0; 1507 rxm->vlan_tci = 0; 1508 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1509 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1510 1511 rxm->port = rxvq->port_id; 1512 1513 rx_pkts[nb_rx] = rxm; 1514 prev = rxm; 1515 1516 if (vq->hw->has_rx_offload && 1517 virtio_rx_offload(rxm, &header->hdr) < 0) { 1518 virtio_discard_rxbuf_inorder(vq, rxm); 1519 rxvq->stats.errors++; 1520 continue; 1521 } 1522 1523 if (hw->vlan_strip) 1524 rte_vlan_strip(rx_pkts[nb_rx]); 1525 1526 seg_res = seg_num - 1; 1527 1528 /* Merge remaining segments */ 1529 while (seg_res != 0 && i < (num - 1)) { 1530 i++; 1531 1532 rxm = rcv_pkts[i]; 1533 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1534 rxm->pkt_len = (uint32_t)(len[i]); 1535 rxm->data_len = (uint16_t)(len[i]); 1536 1537 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1538 1539 prev->next = rxm; 1540 prev = rxm; 1541 seg_res -= 1; 1542 } 1543 1544 if (!seg_res) { 1545 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1546 nb_rx++; 1547 } 1548 } 1549 1550 /* Last packet still need merge segments */ 1551 while (seg_res != 0) { 1552 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1553 VIRTIO_MBUF_BURST_SZ); 1554 1555 if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { 1556 virtio_rmb(hw->weak_barriers); 1557 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1558 rcv_cnt); 1559 uint16_t extra_idx = 0; 1560 1561 rcv_cnt = num; 1562 while (extra_idx < rcv_cnt) { 1563 rxm = rcv_pkts[extra_idx]; 1564 rxm->data_off = 1565 RTE_PKTMBUF_HEADROOM - hdr_size; 1566 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1567 rxm->data_len = (uint16_t)(len[extra_idx]); 1568 prev->next = rxm; 1569 prev = rxm; 1570 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1571 extra_idx += 1; 1572 }; 1573 seg_res -= rcv_cnt; 1574 1575 if (!seg_res) { 1576 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1577 nb_rx++; 1578 } 1579 } else { 1580 PMD_RX_LOG(ERR, 1581 "No enough segments for packet."); 1582 rte_pktmbuf_free(rx_pkts[nb_rx]); 1583 rxvq->stats.errors++; 1584 break; 1585 } 1586 } 1587 1588 rxvq->stats.packets += nb_rx; 1589 1590 /* Allocate new mbuf for the used descriptor */ 1591 1592 if (likely(!virtqueue_full(vq))) { 1593 /* free_cnt may include mrg descs */ 1594 uint16_t free_cnt = vq->vq_free_cnt; 1595 struct rte_mbuf *new_pkts[free_cnt]; 1596 1597 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1598 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1599 free_cnt); 1600 if (unlikely(error)) { 1601 for (i = 0; i < free_cnt; i++) 1602 rte_pktmbuf_free(new_pkts[i]); 1603 } 1604 nb_enqueued += free_cnt; 1605 } else { 1606 struct rte_eth_dev *dev = 1607 &rte_eth_devices[rxvq->port_id]; 1608 dev->data->rx_mbuf_alloc_failed += free_cnt; 1609 } 1610 } 1611 1612 if (likely(nb_enqueued)) { 1613 vq_update_avail_idx(vq); 1614 1615 if (unlikely(virtqueue_kick_prepare(vq))) { 1616 virtqueue_notify(vq); 1617 PMD_RX_LOG(DEBUG, "Notified"); 1618 } 1619 } 1620 1621 return nb_rx; 1622 } 1623 1624 uint16_t 1625 virtio_recv_mergeable_pkts(void *rx_queue, 1626 struct rte_mbuf **rx_pkts, 1627 uint16_t nb_pkts) 1628 { 1629 struct virtnet_rx *rxvq = rx_queue; 1630 struct virtqueue *vq = rxvq->vq; 1631 struct virtio_hw *hw = vq->hw; 1632 struct rte_mbuf *rxm; 1633 struct rte_mbuf *prev = NULL; 1634 uint16_t nb_used, num, nb_rx = 0; 1635 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1636 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1637 int error; 1638 uint32_t nb_enqueued = 0; 1639 uint32_t seg_num = 0; 1640 uint32_t seg_res = 0; 1641 uint32_t hdr_size = hw->vtnet_hdr_size; 1642 int32_t i; 1643 1644 if (unlikely(hw->started == 0)) 1645 return nb_rx; 1646 1647 nb_used = VIRTQUEUE_NUSED(vq); 1648 1649 virtio_rmb(hw->weak_barriers); 1650 1651 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1652 1653 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1654 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1655 num = VIRTIO_MBUF_BURST_SZ; 1656 if (likely(num > DESC_PER_CACHELINE)) 1657 num = num - ((vq->vq_used_cons_idx + num) % 1658 DESC_PER_CACHELINE); 1659 1660 1661 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1662 1663 for (i = 0; i < num; i++) { 1664 struct virtio_net_hdr_mrg_rxbuf *header; 1665 1666 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1667 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1668 1669 rxm = rcv_pkts[i]; 1670 1671 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1672 PMD_RX_LOG(ERR, "Packet drop"); 1673 nb_enqueued++; 1674 virtio_discard_rxbuf(vq, rxm); 1675 rxvq->stats.errors++; 1676 continue; 1677 } 1678 1679 header = (struct virtio_net_hdr_mrg_rxbuf *) 1680 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1681 - hdr_size); 1682 seg_num = header->num_buffers; 1683 if (seg_num == 0) 1684 seg_num = 1; 1685 1686 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1687 rxm->nb_segs = seg_num; 1688 rxm->ol_flags = 0; 1689 rxm->vlan_tci = 0; 1690 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1691 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1692 1693 rxm->port = rxvq->port_id; 1694 1695 rx_pkts[nb_rx] = rxm; 1696 prev = rxm; 1697 1698 if (hw->has_rx_offload && 1699 virtio_rx_offload(rxm, &header->hdr) < 0) { 1700 virtio_discard_rxbuf(vq, rxm); 1701 rxvq->stats.errors++; 1702 continue; 1703 } 1704 1705 if (hw->vlan_strip) 1706 rte_vlan_strip(rx_pkts[nb_rx]); 1707 1708 seg_res = seg_num - 1; 1709 1710 /* Merge remaining segments */ 1711 while (seg_res != 0 && i < (num - 1)) { 1712 i++; 1713 1714 rxm = rcv_pkts[i]; 1715 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1716 rxm->pkt_len = (uint32_t)(len[i]); 1717 rxm->data_len = (uint16_t)(len[i]); 1718 1719 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1720 1721 prev->next = rxm; 1722 prev = rxm; 1723 seg_res -= 1; 1724 } 1725 1726 if (!seg_res) { 1727 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1728 nb_rx++; 1729 } 1730 } 1731 1732 /* Last packet still need merge segments */ 1733 while (seg_res != 0) { 1734 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1735 VIRTIO_MBUF_BURST_SZ); 1736 1737 if (likely(VIRTQUEUE_NUSED(vq) >= rcv_cnt)) { 1738 virtio_rmb(hw->weak_barriers); 1739 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1740 rcv_cnt); 1741 uint16_t extra_idx = 0; 1742 1743 rcv_cnt = num; 1744 while (extra_idx < rcv_cnt) { 1745 rxm = rcv_pkts[extra_idx]; 1746 rxm->data_off = 1747 RTE_PKTMBUF_HEADROOM - hdr_size; 1748 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1749 rxm->data_len = (uint16_t)(len[extra_idx]); 1750 prev->next = rxm; 1751 prev = rxm; 1752 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1753 extra_idx += 1; 1754 }; 1755 seg_res -= rcv_cnt; 1756 1757 if (!seg_res) { 1758 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1759 nb_rx++; 1760 } 1761 } else { 1762 PMD_RX_LOG(ERR, 1763 "No enough segments for packet."); 1764 rte_pktmbuf_free(rx_pkts[nb_rx]); 1765 rxvq->stats.errors++; 1766 break; 1767 } 1768 } 1769 1770 rxvq->stats.packets += nb_rx; 1771 1772 /* Allocate new mbuf for the used descriptor */ 1773 if (likely(!virtqueue_full(vq))) { 1774 /* free_cnt may include mrg descs */ 1775 uint16_t free_cnt = vq->vq_free_cnt; 1776 struct rte_mbuf *new_pkts[free_cnt]; 1777 1778 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1779 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1780 free_cnt); 1781 if (unlikely(error)) { 1782 for (i = 0; i < free_cnt; i++) 1783 rte_pktmbuf_free(new_pkts[i]); 1784 } 1785 nb_enqueued += free_cnt; 1786 } else { 1787 struct rte_eth_dev *dev = 1788 &rte_eth_devices[rxvq->port_id]; 1789 dev->data->rx_mbuf_alloc_failed += free_cnt; 1790 } 1791 } 1792 1793 if (likely(nb_enqueued)) { 1794 vq_update_avail_idx(vq); 1795 1796 if (unlikely(virtqueue_kick_prepare(vq))) { 1797 virtqueue_notify(vq); 1798 PMD_RX_LOG(DEBUG, "Notified"); 1799 } 1800 } 1801 1802 return nb_rx; 1803 } 1804 1805 uint16_t 1806 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1807 struct rte_mbuf **rx_pkts, 1808 uint16_t nb_pkts) 1809 { 1810 struct virtnet_rx *rxvq = rx_queue; 1811 struct virtqueue *vq = rxvq->vq; 1812 struct virtio_hw *hw = vq->hw; 1813 struct rte_mbuf *rxm; 1814 struct rte_mbuf *prev = NULL; 1815 uint16_t num, nb_rx = 0; 1816 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1817 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1818 uint32_t nb_enqueued = 0; 1819 uint32_t seg_num = 0; 1820 uint32_t seg_res = 0; 1821 uint32_t hdr_size = hw->vtnet_hdr_size; 1822 int32_t i; 1823 int error; 1824 1825 if (unlikely(hw->started == 0)) 1826 return nb_rx; 1827 1828 1829 num = nb_pkts; 1830 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1831 num = VIRTIO_MBUF_BURST_SZ; 1832 if (likely(num > DESC_PER_CACHELINE)) 1833 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1834 1835 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1836 1837 for (i = 0; i < num; i++) { 1838 struct virtio_net_hdr_mrg_rxbuf *header; 1839 1840 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1841 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1842 1843 rxm = rcv_pkts[i]; 1844 1845 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1846 PMD_RX_LOG(ERR, "Packet drop"); 1847 nb_enqueued++; 1848 virtio_discard_rxbuf(vq, rxm); 1849 rxvq->stats.errors++; 1850 continue; 1851 } 1852 1853 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1854 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1855 seg_num = header->num_buffers; 1856 1857 if (seg_num == 0) 1858 seg_num = 1; 1859 1860 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1861 rxm->nb_segs = seg_num; 1862 rxm->ol_flags = 0; 1863 rxm->vlan_tci = 0; 1864 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1865 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1866 1867 rxm->port = rxvq->port_id; 1868 rx_pkts[nb_rx] = rxm; 1869 prev = rxm; 1870 1871 if (hw->has_rx_offload && 1872 virtio_rx_offload(rxm, &header->hdr) < 0) { 1873 virtio_discard_rxbuf(vq, rxm); 1874 rxvq->stats.errors++; 1875 continue; 1876 } 1877 1878 if (hw->vlan_strip) 1879 rte_vlan_strip(rx_pkts[nb_rx]); 1880 1881 seg_res = seg_num - 1; 1882 1883 /* Merge remaining segments */ 1884 while (seg_res != 0 && i < (num - 1)) { 1885 i++; 1886 1887 rxm = rcv_pkts[i]; 1888 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1889 rxm->pkt_len = (uint32_t)(len[i]); 1890 rxm->data_len = (uint16_t)(len[i]); 1891 1892 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1893 1894 prev->next = rxm; 1895 prev = rxm; 1896 seg_res -= 1; 1897 } 1898 1899 if (!seg_res) { 1900 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1901 nb_rx++; 1902 } 1903 } 1904 1905 /* Last packet still need merge segments */ 1906 while (seg_res != 0) { 1907 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1908 VIRTIO_MBUF_BURST_SZ); 1909 uint16_t extra_idx = 0; 1910 1911 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1912 len, rcv_cnt); 1913 if (unlikely(rcv_cnt == 0)) { 1914 PMD_RX_LOG(ERR, "No enough segments for packet."); 1915 rte_pktmbuf_free(rx_pkts[nb_rx]); 1916 rxvq->stats.errors++; 1917 break; 1918 } 1919 1920 while (extra_idx < rcv_cnt) { 1921 rxm = rcv_pkts[extra_idx]; 1922 1923 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1924 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1925 rxm->data_len = (uint16_t)(len[extra_idx]); 1926 1927 prev->next = rxm; 1928 prev = rxm; 1929 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1930 extra_idx += 1; 1931 } 1932 seg_res -= rcv_cnt; 1933 if (!seg_res) { 1934 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1935 nb_rx++; 1936 } 1937 } 1938 1939 rxvq->stats.packets += nb_rx; 1940 1941 /* Allocate new mbuf for the used descriptor */ 1942 if (likely(!virtqueue_full(vq))) { 1943 /* free_cnt may include mrg descs */ 1944 uint16_t free_cnt = vq->vq_free_cnt; 1945 struct rte_mbuf *new_pkts[free_cnt]; 1946 1947 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1948 error = virtqueue_enqueue_recv_refill_packed(vq, 1949 new_pkts, free_cnt); 1950 if (unlikely(error)) { 1951 for (i = 0; i < free_cnt; i++) 1952 rte_pktmbuf_free(new_pkts[i]); 1953 } 1954 nb_enqueued += free_cnt; 1955 } else { 1956 struct rte_eth_dev *dev = 1957 &rte_eth_devices[rxvq->port_id]; 1958 dev->data->rx_mbuf_alloc_failed += free_cnt; 1959 } 1960 } 1961 1962 if (likely(nb_enqueued)) { 1963 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1964 virtqueue_notify(vq); 1965 PMD_RX_LOG(DEBUG, "Notified"); 1966 } 1967 } 1968 1969 return nb_rx; 1970 } 1971 1972 uint16_t 1973 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1974 uint16_t nb_pkts) 1975 { 1976 uint16_t nb_tx; 1977 int error; 1978 1979 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1980 struct rte_mbuf *m = tx_pkts[nb_tx]; 1981 1982 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1983 error = rte_validate_tx_offload(m); 1984 if (unlikely(error)) { 1985 rte_errno = -error; 1986 break; 1987 } 1988 #endif 1989 1990 /* Do VLAN tag insertion */ 1991 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) { 1992 error = rte_vlan_insert(&m); 1993 /* rte_vlan_insert() may change pointer 1994 * even in the case of failure 1995 */ 1996 tx_pkts[nb_tx] = m; 1997 1998 if (unlikely(error)) { 1999 rte_errno = -error; 2000 break; 2001 } 2002 } 2003 2004 error = rte_net_intel_cksum_prepare(m); 2005 if (unlikely(error)) { 2006 rte_errno = -error; 2007 break; 2008 } 2009 2010 if (m->ol_flags & PKT_TX_TCP_SEG) 2011 virtio_tso_fix_cksum(m); 2012 } 2013 2014 return nb_tx; 2015 } 2016 2017 uint16_t 2018 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 2019 uint16_t nb_pkts) 2020 { 2021 struct virtnet_tx *txvq = tx_queue; 2022 struct virtqueue *vq = txvq->vq; 2023 struct virtio_hw *hw = vq->hw; 2024 uint16_t hdr_size = hw->vtnet_hdr_size; 2025 uint16_t nb_tx = 0; 2026 bool in_order = hw->use_inorder_tx; 2027 2028 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 2029 return nb_tx; 2030 2031 if (unlikely(nb_pkts < 1)) 2032 return nb_pkts; 2033 2034 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 2035 2036 if (nb_pkts > vq->vq_free_cnt) 2037 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 2038 in_order); 2039 2040 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 2041 struct rte_mbuf *txm = tx_pkts[nb_tx]; 2042 int can_push = 0, slots, need; 2043 2044 /* optimize ring usage */ 2045 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 2046 vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && 2047 rte_mbuf_refcnt_read(txm) == 1 && 2048 RTE_MBUF_DIRECT(txm) && 2049 txm->nb_segs == 1 && 2050 rte_pktmbuf_headroom(txm) >= hdr_size && 2051 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 2052 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 2053 can_push = 1; 2054 2055 /* How many main ring entries are needed to this Tx? 2056 * any_layout => number of segments 2057 * default => number of segments + 1 2058 */ 2059 slots = txm->nb_segs + !can_push; 2060 need = slots - vq->vq_free_cnt; 2061 2062 /* Positive value indicates it need free vring descriptors */ 2063 if (unlikely(need > 0)) { 2064 virtio_xmit_cleanup_packed(vq, need, in_order); 2065 need = slots - vq->vq_free_cnt; 2066 if (unlikely(need > 0)) { 2067 PMD_TX_LOG(ERR, 2068 "No free tx descriptors to transmit"); 2069 break; 2070 } 2071 } 2072 2073 /* Enqueue Packet buffers */ 2074 if (can_push) 2075 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 2076 else 2077 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 0, 2078 in_order); 2079 2080 virtio_update_packet_stats(&txvq->stats, txm); 2081 } 2082 2083 txvq->stats.packets += nb_tx; 2084 2085 if (likely(nb_tx)) { 2086 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 2087 virtqueue_notify(vq); 2088 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2089 } 2090 } 2091 2092 return nb_tx; 2093 } 2094 2095 uint16_t 2096 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2097 { 2098 struct virtnet_tx *txvq = tx_queue; 2099 struct virtqueue *vq = txvq->vq; 2100 struct virtio_hw *hw = vq->hw; 2101 uint16_t hdr_size = hw->vtnet_hdr_size; 2102 uint16_t nb_used, nb_tx = 0; 2103 2104 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 2105 return nb_tx; 2106 2107 if (unlikely(nb_pkts < 1)) 2108 return nb_pkts; 2109 2110 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 2111 nb_used = VIRTQUEUE_NUSED(vq); 2112 2113 virtio_rmb(hw->weak_barriers); 2114 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 2115 virtio_xmit_cleanup(vq, nb_used); 2116 2117 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 2118 struct rte_mbuf *txm = tx_pkts[nb_tx]; 2119 int can_push = 0, use_indirect = 0, slots, need; 2120 2121 /* optimize ring usage */ 2122 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 2123 vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && 2124 rte_mbuf_refcnt_read(txm) == 1 && 2125 RTE_MBUF_DIRECT(txm) && 2126 txm->nb_segs == 1 && 2127 rte_pktmbuf_headroom(txm) >= hdr_size && 2128 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 2129 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 2130 can_push = 1; 2131 else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 2132 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 2133 use_indirect = 1; 2134 2135 /* How many main ring entries are needed to this Tx? 2136 * any_layout => number of segments 2137 * indirect => 1 2138 * default => number of segments + 1 2139 */ 2140 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 2141 need = slots - vq->vq_free_cnt; 2142 2143 /* Positive value indicates it need free vring descriptors */ 2144 if (unlikely(need > 0)) { 2145 nb_used = VIRTQUEUE_NUSED(vq); 2146 virtio_rmb(hw->weak_barriers); 2147 need = RTE_MIN(need, (int)nb_used); 2148 2149 virtio_xmit_cleanup(vq, need); 2150 need = slots - vq->vq_free_cnt; 2151 if (unlikely(need > 0)) { 2152 PMD_TX_LOG(ERR, 2153 "No free tx descriptors to transmit"); 2154 break; 2155 } 2156 } 2157 2158 /* Enqueue Packet buffers */ 2159 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 2160 can_push, 0); 2161 2162 virtio_update_packet_stats(&txvq->stats, txm); 2163 } 2164 2165 txvq->stats.packets += nb_tx; 2166 2167 if (likely(nb_tx)) { 2168 vq_update_avail_idx(vq); 2169 2170 if (unlikely(virtqueue_kick_prepare(vq))) { 2171 virtqueue_notify(vq); 2172 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2173 } 2174 } 2175 2176 return nb_tx; 2177 } 2178 2179 static __rte_always_inline int 2180 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 2181 { 2182 uint16_t nb_used, nb_clean, nb_descs; 2183 struct virtio_hw *hw = vq->hw; 2184 2185 nb_descs = vq->vq_free_cnt + need; 2186 nb_used = VIRTQUEUE_NUSED(vq); 2187 virtio_rmb(hw->weak_barriers); 2188 nb_clean = RTE_MIN(need, (int)nb_used); 2189 2190 virtio_xmit_cleanup_inorder(vq, nb_clean); 2191 2192 return nb_descs - vq->vq_free_cnt; 2193 } 2194 2195 uint16_t 2196 virtio_xmit_pkts_inorder(void *tx_queue, 2197 struct rte_mbuf **tx_pkts, 2198 uint16_t nb_pkts) 2199 { 2200 struct virtnet_tx *txvq = tx_queue; 2201 struct virtqueue *vq = txvq->vq; 2202 struct virtio_hw *hw = vq->hw; 2203 uint16_t hdr_size = hw->vtnet_hdr_size; 2204 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 2205 struct rte_mbuf *inorder_pkts[nb_pkts]; 2206 int need; 2207 2208 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 2209 return nb_tx; 2210 2211 if (unlikely(nb_pkts < 1)) 2212 return nb_pkts; 2213 2214 VIRTQUEUE_DUMP(vq); 2215 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 2216 nb_used = VIRTQUEUE_NUSED(vq); 2217 2218 virtio_rmb(hw->weak_barriers); 2219 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 2220 virtio_xmit_cleanup_inorder(vq, nb_used); 2221 2222 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 2223 struct rte_mbuf *txm = tx_pkts[nb_tx]; 2224 int slots; 2225 2226 /* optimize ring usage */ 2227 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 2228 vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) && 2229 rte_mbuf_refcnt_read(txm) == 1 && 2230 RTE_MBUF_DIRECT(txm) && 2231 txm->nb_segs == 1 && 2232 rte_pktmbuf_headroom(txm) >= hdr_size && 2233 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 2234 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) { 2235 inorder_pkts[nb_inorder_pkts] = txm; 2236 nb_inorder_pkts++; 2237 2238 continue; 2239 } 2240 2241 if (nb_inorder_pkts) { 2242 need = nb_inorder_pkts - vq->vq_free_cnt; 2243 if (unlikely(need > 0)) { 2244 need = virtio_xmit_try_cleanup_inorder(vq, 2245 need); 2246 if (unlikely(need > 0)) { 2247 PMD_TX_LOG(ERR, 2248 "No free tx descriptors to " 2249 "transmit"); 2250 break; 2251 } 2252 } 2253 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2254 nb_inorder_pkts); 2255 nb_inorder_pkts = 0; 2256 } 2257 2258 slots = txm->nb_segs + 1; 2259 need = slots - vq->vq_free_cnt; 2260 if (unlikely(need > 0)) { 2261 need = virtio_xmit_try_cleanup_inorder(vq, slots); 2262 2263 if (unlikely(need > 0)) { 2264 PMD_TX_LOG(ERR, 2265 "No free tx descriptors to transmit"); 2266 break; 2267 } 2268 } 2269 /* Enqueue Packet buffers */ 2270 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 2271 2272 virtio_update_packet_stats(&txvq->stats, txm); 2273 } 2274 2275 /* Transmit all inorder packets */ 2276 if (nb_inorder_pkts) { 2277 need = nb_inorder_pkts - vq->vq_free_cnt; 2278 if (unlikely(need > 0)) { 2279 need = virtio_xmit_try_cleanup_inorder(vq, 2280 need); 2281 if (unlikely(need > 0)) { 2282 PMD_TX_LOG(ERR, 2283 "No free tx descriptors to transmit"); 2284 nb_inorder_pkts = vq->vq_free_cnt; 2285 nb_tx -= need; 2286 } 2287 } 2288 2289 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2290 nb_inorder_pkts); 2291 } 2292 2293 txvq->stats.packets += nb_tx; 2294 2295 if (likely(nb_tx)) { 2296 vq_update_avail_idx(vq); 2297 2298 if (unlikely(virtqueue_kick_prepare(vq))) { 2299 virtqueue_notify(vq); 2300 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2301 } 2302 } 2303 2304 VIRTQUEUE_DUMP(vq); 2305 2306 return nb_tx; 2307 } 2308