1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_cycles.h> 12 #include <rte_memory.h> 13 #include <rte_branch_prediction.h> 14 #include <rte_mempool.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_ether.h> 18 #include <ethdev_driver.h> 19 #include <rte_prefetch.h> 20 #include <rte_string_fns.h> 21 #include <rte_errno.h> 22 #include <rte_byteorder.h> 23 #include <rte_net.h> 24 #include <rte_ip.h> 25 #include <rte_udp.h> 26 #include <rte_tcp.h> 27 28 #include "virtio_logs.h" 29 #include "virtio_ethdev.h" 30 #include "virtio.h" 31 #include "virtqueue.h" 32 #include "virtio_rxtx.h" 33 #include "virtio_rxtx_simple.h" 34 #include "virtio_ring.h" 35 36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 38 #else 39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 40 #endif 41 42 int 43 virtio_dev_rx_queue_done(void *rxq, uint16_t offset) 44 { 45 struct virtnet_rx *rxvq = rxq; 46 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 47 48 return virtqueue_nused(vq) >= offset; 49 } 50 51 void 52 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 53 { 54 vq->vq_free_cnt += num; 55 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 56 } 57 58 void 59 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 60 { 61 struct vring_desc *dp, *dp_tail; 62 struct vq_desc_extra *dxp; 63 uint16_t desc_idx_last = desc_idx; 64 65 dp = &vq->vq_split.ring.desc[desc_idx]; 66 dxp = &vq->vq_descx[desc_idx]; 67 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 68 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 69 while (dp->flags & VRING_DESC_F_NEXT) { 70 desc_idx_last = dp->next; 71 dp = &vq->vq_split.ring.desc[dp->next]; 72 } 73 } 74 dxp->ndescs = 0; 75 76 /* 77 * We must append the existing free chain, if any, to the end of 78 * newly freed chain. If the virtqueue was completely used, then 79 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 80 */ 81 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 82 vq->vq_desc_head_idx = desc_idx; 83 } else { 84 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 85 dp_tail->next = desc_idx; 86 } 87 88 vq->vq_desc_tail_idx = desc_idx_last; 89 dp->next = VQ_RING_DESC_CHAIN_END; 90 } 91 92 void 93 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) 94 { 95 uint32_t s = mbuf->pkt_len; 96 struct rte_ether_addr *ea; 97 98 stats->bytes += s; 99 100 if (s == 64) { 101 stats->size_bins[1]++; 102 } else if (s > 64 && s < 1024) { 103 uint32_t bin; 104 105 /* count zeros, and offset into correct bin */ 106 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 107 stats->size_bins[bin]++; 108 } else { 109 if (s < 64) 110 stats->size_bins[0]++; 111 else if (s < 1519) 112 stats->size_bins[6]++; 113 else 114 stats->size_bins[7]++; 115 } 116 117 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 118 if (rte_is_multicast_ether_addr(ea)) { 119 if (rte_is_broadcast_ether_addr(ea)) 120 stats->broadcast++; 121 else 122 stats->multicast++; 123 } 124 } 125 126 static inline void 127 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 128 { 129 VIRTIO_DUMP_PACKET(m, m->data_len); 130 131 virtio_update_packet_stats(&rxvq->stats, m); 132 } 133 134 static uint16_t 135 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 136 struct rte_mbuf **rx_pkts, 137 uint32_t *len, 138 uint16_t num) 139 { 140 struct rte_mbuf *cookie; 141 uint16_t used_idx; 142 uint16_t id; 143 struct vring_packed_desc *desc; 144 uint16_t i; 145 146 desc = vq->vq_packed.ring.desc; 147 148 for (i = 0; i < num; i++) { 149 used_idx = vq->vq_used_cons_idx; 150 /* desc_is_used has a load-acquire or rte_io_rmb inside 151 * and wait for used desc in virtqueue. 152 */ 153 if (!desc_is_used(&desc[used_idx], vq)) 154 return i; 155 len[i] = desc[used_idx].len; 156 id = desc[used_idx].id; 157 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 158 if (unlikely(cookie == NULL)) { 159 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 160 vq->vq_used_cons_idx); 161 break; 162 } 163 rte_prefetch0(cookie); 164 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 165 rx_pkts[i] = cookie; 166 167 vq->vq_free_cnt++; 168 vq->vq_used_cons_idx++; 169 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 170 vq->vq_used_cons_idx -= vq->vq_nentries; 171 vq->vq_packed.used_wrap_counter ^= 1; 172 } 173 } 174 175 return i; 176 } 177 178 static uint16_t 179 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 180 uint32_t *len, uint16_t num) 181 { 182 struct vring_used_elem *uep; 183 struct rte_mbuf *cookie; 184 uint16_t used_idx, desc_idx; 185 uint16_t i; 186 187 /* Caller does the check */ 188 for (i = 0; i < num ; i++) { 189 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 190 uep = &vq->vq_split.ring.used->ring[used_idx]; 191 desc_idx = (uint16_t) uep->id; 192 len[i] = uep->len; 193 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 194 195 if (unlikely(cookie == NULL)) { 196 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 197 vq->vq_used_cons_idx); 198 break; 199 } 200 201 rte_prefetch0(cookie); 202 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 203 rx_pkts[i] = cookie; 204 vq->vq_used_cons_idx++; 205 vq_ring_free_chain(vq, desc_idx); 206 vq->vq_descx[desc_idx].cookie = NULL; 207 } 208 209 return i; 210 } 211 212 static uint16_t 213 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 214 struct rte_mbuf **rx_pkts, 215 uint32_t *len, 216 uint16_t num) 217 { 218 struct vring_used_elem *uep; 219 struct rte_mbuf *cookie; 220 uint16_t used_idx = 0; 221 uint16_t i; 222 223 if (unlikely(num == 0)) 224 return 0; 225 226 for (i = 0; i < num; i++) { 227 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 228 /* Desc idx same as used idx */ 229 uep = &vq->vq_split.ring.used->ring[used_idx]; 230 len[i] = uep->len; 231 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 232 233 if (unlikely(cookie == NULL)) { 234 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 235 vq->vq_used_cons_idx); 236 break; 237 } 238 239 rte_prefetch0(cookie); 240 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 241 rx_pkts[i] = cookie; 242 vq->vq_used_cons_idx++; 243 vq->vq_descx[used_idx].cookie = NULL; 244 } 245 246 vq_ring_free_inorder(vq, used_idx, i); 247 return i; 248 } 249 250 static inline int 251 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 252 struct rte_mbuf **cookies, 253 uint16_t num) 254 { 255 struct vq_desc_extra *dxp; 256 struct virtio_hw *hw = vq->hw; 257 struct vring_desc *start_dp; 258 uint16_t head_idx, idx, i = 0; 259 260 if (unlikely(vq->vq_free_cnt == 0)) 261 return -ENOSPC; 262 if (unlikely(vq->vq_free_cnt < num)) 263 return -EMSGSIZE; 264 265 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 266 start_dp = vq->vq_split.ring.desc; 267 268 while (i < num) { 269 idx = head_idx & (vq->vq_nentries - 1); 270 dxp = &vq->vq_descx[idx]; 271 dxp->cookie = (void *)cookies[i]; 272 dxp->ndescs = 1; 273 274 start_dp[idx].addr = cookies[i]->buf_iova + 275 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 276 start_dp[idx].len = cookies[i]->buf_len - 277 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 278 start_dp[idx].flags = VRING_DESC_F_WRITE; 279 280 vq_update_avail_ring(vq, idx); 281 head_idx++; 282 i++; 283 } 284 285 vq->vq_desc_head_idx += num; 286 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 287 return 0; 288 } 289 290 static inline int 291 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 292 uint16_t num) 293 { 294 struct vq_desc_extra *dxp; 295 struct virtio_hw *hw = vq->hw; 296 struct vring_desc *start_dp = vq->vq_split.ring.desc; 297 uint16_t idx, i; 298 299 if (unlikely(vq->vq_free_cnt == 0)) 300 return -ENOSPC; 301 if (unlikely(vq->vq_free_cnt < num)) 302 return -EMSGSIZE; 303 304 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 305 return -EFAULT; 306 307 for (i = 0; i < num; i++) { 308 idx = vq->vq_desc_head_idx; 309 dxp = &vq->vq_descx[idx]; 310 dxp->cookie = (void *)cookie[i]; 311 dxp->ndescs = 1; 312 313 start_dp[idx].addr = cookie[i]->buf_iova + 314 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 315 start_dp[idx].len = cookie[i]->buf_len - 316 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 317 start_dp[idx].flags = VRING_DESC_F_WRITE; 318 vq->vq_desc_head_idx = start_dp[idx].next; 319 vq_update_avail_ring(vq, idx); 320 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 321 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 322 break; 323 } 324 } 325 326 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 327 328 return 0; 329 } 330 331 static inline void 332 virtqueue_refill_single_packed(struct virtqueue *vq, 333 struct vring_packed_desc *dp, 334 struct rte_mbuf *cookie) 335 { 336 uint16_t flags = vq->vq_packed.cached_flags; 337 struct virtio_hw *hw = vq->hw; 338 339 dp->addr = cookie->buf_iova + 340 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 341 dp->len = cookie->buf_len - 342 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 343 344 virtqueue_store_flags_packed(dp, flags, 345 hw->weak_barriers); 346 347 if (++vq->vq_avail_idx >= vq->vq_nentries) { 348 vq->vq_avail_idx -= vq->vq_nentries; 349 vq->vq_packed.cached_flags ^= 350 VRING_PACKED_DESC_F_AVAIL_USED; 351 flags = vq->vq_packed.cached_flags; 352 } 353 } 354 355 static inline int 356 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq, 357 struct rte_mbuf **cookie, uint16_t num) 358 { 359 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 360 struct vq_desc_extra *dxp; 361 uint16_t idx; 362 int i; 363 364 if (unlikely(vq->vq_free_cnt == 0)) 365 return -ENOSPC; 366 if (unlikely(vq->vq_free_cnt < num)) 367 return -EMSGSIZE; 368 369 for (i = 0; i < num; i++) { 370 idx = vq->vq_avail_idx; 371 dxp = &vq->vq_descx[idx]; 372 dxp->cookie = (void *)cookie[i]; 373 dxp->ndescs = 1; 374 375 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 376 } 377 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 378 return 0; 379 } 380 381 static inline int 382 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 383 struct rte_mbuf **cookie, uint16_t num) 384 { 385 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 386 struct vq_desc_extra *dxp; 387 uint16_t idx, did; 388 int i; 389 390 if (unlikely(vq->vq_free_cnt == 0)) 391 return -ENOSPC; 392 if (unlikely(vq->vq_free_cnt < num)) 393 return -EMSGSIZE; 394 395 for (i = 0; i < num; i++) { 396 idx = vq->vq_avail_idx; 397 did = start_dp[idx].id; 398 dxp = &vq->vq_descx[did]; 399 dxp->cookie = (void *)cookie[i]; 400 dxp->ndescs = 1; 401 402 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 403 } 404 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 405 return 0; 406 } 407 408 /* When doing TSO, the IP length is not included in the pseudo header 409 * checksum of the packet given to the PMD, but for virtio it is 410 * expected. 411 */ 412 static void 413 virtio_tso_fix_cksum(struct rte_mbuf *m) 414 { 415 /* common case: header is not fragmented */ 416 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 417 m->l4_len)) { 418 struct rte_ipv4_hdr *iph; 419 struct rte_ipv6_hdr *ip6h; 420 struct rte_tcp_hdr *th; 421 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; 422 uint32_t tmp; 423 424 iph = rte_pktmbuf_mtod_offset(m, 425 struct rte_ipv4_hdr *, m->l2_len); 426 th = RTE_PTR_ADD(iph, m->l3_len); 427 if ((iph->version_ihl >> 4) == 4) { 428 iph->hdr_checksum = 0; 429 iph->hdr_checksum = rte_ipv4_cksum(iph); 430 ip_len = iph->total_length; 431 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - 432 m->l3_len); 433 } else { 434 ip6h = (struct rte_ipv6_hdr *)iph; 435 ip_paylen = ip6h->payload_len; 436 } 437 438 /* calculate the new phdr checksum not including ip_paylen */ 439 prev_cksum = th->cksum; 440 tmp = prev_cksum; 441 tmp += ip_paylen; 442 tmp = (tmp & 0xffff) + (tmp >> 16); 443 new_cksum = tmp; 444 445 /* replace it in the packet */ 446 th->cksum = new_cksum; 447 } 448 } 449 450 451 452 453 static inline void 454 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 455 struct rte_mbuf **cookies, 456 uint16_t num) 457 { 458 struct vq_desc_extra *dxp; 459 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 460 struct vring_desc *start_dp; 461 struct virtio_net_hdr *hdr; 462 uint16_t idx; 463 int16_t head_size = vq->hw->vtnet_hdr_size; 464 uint16_t i = 0; 465 466 idx = vq->vq_desc_head_idx; 467 start_dp = vq->vq_split.ring.desc; 468 469 while (i < num) { 470 idx = idx & (vq->vq_nentries - 1); 471 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 472 dxp->cookie = (void *)cookies[i]; 473 dxp->ndescs = 1; 474 virtio_update_packet_stats(&txvq->stats, cookies[i]); 475 476 hdr = rte_pktmbuf_mtod_offset(cookies[i], 477 struct virtio_net_hdr *, -head_size); 478 479 /* if offload disabled, hdr is not zeroed yet, do it now */ 480 if (!vq->hw->has_tx_offload) 481 virtqueue_clear_net_hdr(hdr); 482 else 483 virtqueue_xmit_offload(hdr, cookies[i]); 484 485 start_dp[idx].addr = rte_mbuf_data_iova(cookies[i]) - head_size; 486 start_dp[idx].len = cookies[i]->data_len + head_size; 487 start_dp[idx].flags = 0; 488 489 490 vq_update_avail_ring(vq, idx); 491 492 idx++; 493 i++; 494 }; 495 496 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 497 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 498 } 499 500 static inline void 501 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 502 struct rte_mbuf *cookie, 503 int in_order) 504 { 505 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 506 struct vring_packed_desc *dp; 507 struct vq_desc_extra *dxp; 508 uint16_t idx, id, flags; 509 int16_t head_size = vq->hw->vtnet_hdr_size; 510 struct virtio_net_hdr *hdr; 511 512 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 513 idx = vq->vq_avail_idx; 514 dp = &vq->vq_packed.ring.desc[idx]; 515 516 dxp = &vq->vq_descx[id]; 517 dxp->ndescs = 1; 518 dxp->cookie = cookie; 519 520 flags = vq->vq_packed.cached_flags; 521 522 /* prepend cannot fail, checked by caller */ 523 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 524 -head_size); 525 526 /* if offload disabled, hdr is not zeroed yet, do it now */ 527 if (!vq->hw->has_tx_offload) 528 virtqueue_clear_net_hdr(hdr); 529 else 530 virtqueue_xmit_offload(hdr, cookie); 531 532 dp->addr = rte_mbuf_data_iova(cookie) - head_size; 533 dp->len = cookie->data_len + head_size; 534 dp->id = id; 535 536 if (++vq->vq_avail_idx >= vq->vq_nentries) { 537 vq->vq_avail_idx -= vq->vq_nentries; 538 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 539 } 540 541 vq->vq_free_cnt--; 542 543 if (!in_order) { 544 vq->vq_desc_head_idx = dxp->next; 545 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 546 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 547 } 548 549 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 550 } 551 552 static inline void 553 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 554 uint16_t needed, int use_indirect, int can_push, 555 int in_order) 556 { 557 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 558 struct vq_desc_extra *dxp; 559 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 560 struct vring_desc *start_dp; 561 uint16_t seg_num = cookie->nb_segs; 562 uint16_t head_idx, idx; 563 int16_t head_size = vq->hw->vtnet_hdr_size; 564 bool prepend_header = false; 565 struct virtio_net_hdr *hdr; 566 567 head_idx = vq->vq_desc_head_idx; 568 idx = head_idx; 569 if (in_order) 570 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 571 else 572 dxp = &vq->vq_descx[idx]; 573 dxp->cookie = (void *)cookie; 574 dxp->ndescs = needed; 575 576 start_dp = vq->vq_split.ring.desc; 577 578 if (can_push) { 579 /* prepend cannot fail, checked by caller */ 580 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 581 -head_size); 582 prepend_header = true; 583 584 /* if offload disabled, it is not zeroed below, do it now */ 585 if (!vq->hw->has_tx_offload) 586 virtqueue_clear_net_hdr(hdr); 587 } else if (use_indirect) { 588 /* setup tx ring slot to point to indirect 589 * descriptor list stored in reserved region. 590 * 591 * the first slot in indirect ring is already preset 592 * to point to the header in reserved region 593 */ 594 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 595 RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 596 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 597 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 598 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 599 600 /* loop below will fill in rest of the indirect elements */ 601 start_dp = txr[idx].tx_indir; 602 idx = 1; 603 } else { 604 /* setup first tx ring slot to point to header 605 * stored in reserved region. 606 */ 607 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 608 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 609 start_dp[idx].len = vq->hw->vtnet_hdr_size; 610 start_dp[idx].flags = VRING_DESC_F_NEXT; 611 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 612 613 idx = start_dp[idx].next; 614 } 615 616 if (vq->hw->has_tx_offload) 617 virtqueue_xmit_offload(hdr, cookie); 618 619 do { 620 start_dp[idx].addr = rte_mbuf_data_iova(cookie); 621 start_dp[idx].len = cookie->data_len; 622 if (prepend_header) { 623 start_dp[idx].addr -= head_size; 624 start_dp[idx].len += head_size; 625 prepend_header = false; 626 } 627 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 628 idx = start_dp[idx].next; 629 } while ((cookie = cookie->next) != NULL); 630 631 if (use_indirect) 632 idx = vq->vq_split.ring.desc[head_idx].next; 633 634 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 635 636 vq->vq_desc_head_idx = idx; 637 vq_update_avail_ring(vq, head_idx); 638 639 if (!in_order) { 640 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 641 vq->vq_desc_tail_idx = idx; 642 } 643 } 644 645 void 646 virtio_dev_cq_start(struct rte_eth_dev *dev) 647 { 648 struct virtio_hw *hw = dev->data->dev_private; 649 650 if (hw->cvq) { 651 rte_spinlock_init(&hw->cvq->lock); 652 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); 653 } 654 } 655 656 int 657 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 658 uint16_t queue_idx, 659 uint16_t nb_desc, 660 unsigned int socket_id __rte_unused, 661 const struct rte_eth_rxconf *rx_conf, 662 struct rte_mempool *mp) 663 { 664 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 665 struct virtio_hw *hw = dev->data->dev_private; 666 struct virtqueue *vq = hw->vqs[vq_idx]; 667 struct virtnet_rx *rxvq; 668 uint16_t rx_free_thresh; 669 uint16_t buf_size; 670 const char *error; 671 672 PMD_INIT_FUNC_TRACE(); 673 674 if (rx_conf->rx_deferred_start) { 675 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 676 return -EINVAL; 677 } 678 679 buf_size = virtio_rx_mem_pool_buf_size(mp); 680 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size, 681 hw->rx_ol_scatter, &error)) { 682 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s", 683 queue_idx, error); 684 return -EINVAL; 685 } 686 687 rx_free_thresh = rx_conf->rx_free_thresh; 688 if (rx_free_thresh == 0) 689 rx_free_thresh = 690 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); 691 692 if (rx_free_thresh & 0x3) { 693 RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four." 694 " (rx_free_thresh=%u port=%u queue=%u)\n", 695 rx_free_thresh, dev->data->port_id, queue_idx); 696 return -EINVAL; 697 } 698 699 if (rx_free_thresh >= vq->vq_nentries) { 700 RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the " 701 "number of RX entries (%u)." 702 " (rx_free_thresh=%u port=%u queue=%u)\n", 703 vq->vq_nentries, 704 rx_free_thresh, dev->data->port_id, queue_idx); 705 return -EINVAL; 706 } 707 vq->vq_free_thresh = rx_free_thresh; 708 709 if (nb_desc > vq->vq_nentries) 710 nb_desc = vq->vq_nentries; 711 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 712 713 rxvq = &vq->rxq; 714 rxvq->queue_id = queue_idx; 715 rxvq->mpool = mp; 716 dev->data->rx_queues[queue_idx] = rxvq; 717 718 return 0; 719 } 720 721 int 722 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 723 { 724 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 725 struct virtio_hw *hw = dev->data->dev_private; 726 struct virtqueue *vq = hw->vqs[vq_idx]; 727 struct virtnet_rx *rxvq = &vq->rxq; 728 struct rte_mbuf *m; 729 uint16_t desc_idx; 730 int error, nbufs, i; 731 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 732 733 PMD_INIT_FUNC_TRACE(); 734 735 /* Allocate blank mbufs for the each rx descriptor */ 736 nbufs = 0; 737 738 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 739 for (desc_idx = 0; desc_idx < vq->vq_nentries; 740 desc_idx++) { 741 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 742 vq->vq_split.ring.desc[desc_idx].flags = 743 VRING_DESC_F_WRITE; 744 } 745 746 virtio_rxq_vec_setup(rxvq); 747 } 748 749 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); 750 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) 751 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; 752 753 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 754 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 755 virtio_rxq_rearm_vec(rxvq); 756 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 757 } 758 } else if (!virtio_with_packed_queue(vq->hw) && in_order) { 759 if ((!virtqueue_full(vq))) { 760 uint16_t free_cnt = vq->vq_free_cnt; 761 struct rte_mbuf *pkts[free_cnt]; 762 763 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 764 free_cnt)) { 765 error = virtqueue_enqueue_refill_inorder(vq, 766 pkts, 767 free_cnt); 768 if (unlikely(error)) { 769 for (i = 0; i < free_cnt; i++) 770 rte_pktmbuf_free(pkts[i]); 771 } else { 772 nbufs += free_cnt; 773 } 774 } 775 776 vq_update_avail_idx(vq); 777 } 778 } else { 779 while (!virtqueue_full(vq)) { 780 m = rte_mbuf_raw_alloc(rxvq->mpool); 781 if (m == NULL) 782 break; 783 784 /* Enqueue allocated buffers */ 785 if (virtio_with_packed_queue(vq->hw)) 786 error = virtqueue_enqueue_recv_refill_packed_init(vq, 787 &m, 1); 788 else 789 error = virtqueue_enqueue_recv_refill(vq, 790 &m, 1); 791 if (error) { 792 rte_pktmbuf_free(m); 793 break; 794 } 795 nbufs++; 796 } 797 798 if (!virtio_with_packed_queue(vq->hw)) 799 vq_update_avail_idx(vq); 800 } 801 802 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); 803 804 VIRTQUEUE_DUMP(vq); 805 806 return 0; 807 } 808 809 /* 810 * struct rte_eth_dev *dev: Used to update dev 811 * uint16_t nb_desc: Defaults to values read from config space 812 * unsigned int socket_id: Used to allocate memzone 813 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 814 * uint16_t queue_idx: Just used as an index in dev txq list 815 */ 816 int 817 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 818 uint16_t queue_idx, 819 uint16_t nb_desc, 820 unsigned int socket_id __rte_unused, 821 const struct rte_eth_txconf *tx_conf) 822 { 823 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 824 struct virtio_hw *hw = dev->data->dev_private; 825 struct virtqueue *vq = hw->vqs[vq_idx]; 826 struct virtnet_tx *txvq; 827 uint16_t tx_free_thresh; 828 829 PMD_INIT_FUNC_TRACE(); 830 831 if (tx_conf->tx_deferred_start) { 832 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 833 return -EINVAL; 834 } 835 836 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 837 nb_desc = vq->vq_nentries; 838 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 839 840 txvq = &vq->txq; 841 txvq->queue_id = queue_idx; 842 843 tx_free_thresh = tx_conf->tx_free_thresh; 844 if (tx_free_thresh == 0) 845 tx_free_thresh = 846 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 847 848 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 849 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " 850 "number of TX entries minus 3 (%u)." 851 " (tx_free_thresh=%u port=%u queue=%u)\n", 852 vq->vq_nentries - 3, 853 tx_free_thresh, dev->data->port_id, queue_idx); 854 return -EINVAL; 855 } 856 857 vq->vq_free_thresh = tx_free_thresh; 858 859 dev->data->tx_queues[queue_idx] = txvq; 860 return 0; 861 } 862 863 int 864 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 865 uint16_t queue_idx) 866 { 867 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 868 struct virtio_hw *hw = dev->data->dev_private; 869 struct virtqueue *vq = hw->vqs[vq_idx]; 870 871 PMD_INIT_FUNC_TRACE(); 872 873 if (!virtio_with_packed_queue(hw)) { 874 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 875 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 876 } 877 878 VIRTQUEUE_DUMP(vq); 879 880 return 0; 881 } 882 883 static inline void 884 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 885 { 886 int error; 887 /* 888 * Requeue the discarded mbuf. This should always be 889 * successful since it was just dequeued. 890 */ 891 if (virtio_with_packed_queue(vq->hw)) 892 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 893 else 894 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 895 896 if (unlikely(error)) { 897 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 898 rte_pktmbuf_free(m); 899 } 900 } 901 902 static inline void 903 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 904 { 905 int error; 906 907 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 908 if (unlikely(error)) { 909 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 910 rte_pktmbuf_free(m); 911 } 912 } 913 914 /* Optionally fill offload information in structure */ 915 static inline int 916 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 917 { 918 struct rte_net_hdr_lens hdr_lens; 919 uint32_t hdrlen, ptype; 920 int l4_supported = 0; 921 922 /* nothing to do */ 923 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 924 return 0; 925 926 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; 927 928 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 929 m->packet_type = ptype; 930 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 931 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 932 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 933 l4_supported = 1; 934 935 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 936 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 937 if (hdr->csum_start <= hdrlen && l4_supported) { 938 m->ol_flags |= PKT_RX_L4_CKSUM_NONE; 939 } else { 940 /* Unknown proto or tunnel, do sw cksum. We can assume 941 * the cksum field is in the first segment since the 942 * buffers we provided to the host are large enough. 943 * In case of SCTP, this will be wrong since it's a CRC 944 * but there's nothing we can do. 945 */ 946 uint16_t csum = 0, off; 947 948 if (rte_raw_cksum_mbuf(m, hdr->csum_start, 949 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 950 &csum) < 0) 951 return -EINVAL; 952 if (likely(csum != 0xffff)) 953 csum = ~csum; 954 off = hdr->csum_offset + hdr->csum_start; 955 if (rte_pktmbuf_data_len(m) >= off + 1) 956 *rte_pktmbuf_mtod_offset(m, uint16_t *, 957 off) = csum; 958 } 959 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 960 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 961 } 962 963 /* GSO request, save required information in mbuf */ 964 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 965 /* Check unsupported modes */ 966 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 967 (hdr->gso_size == 0)) { 968 return -EINVAL; 969 } 970 971 /* Update mss lengthes in mbuf */ 972 m->tso_segsz = hdr->gso_size; 973 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 974 case VIRTIO_NET_HDR_GSO_TCPV4: 975 case VIRTIO_NET_HDR_GSO_TCPV6: 976 m->ol_flags |= PKT_RX_LRO | \ 977 PKT_RX_L4_CKSUM_NONE; 978 break; 979 default: 980 return -EINVAL; 981 } 982 } 983 984 return 0; 985 } 986 987 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 988 uint16_t 989 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 990 { 991 struct virtnet_rx *rxvq = rx_queue; 992 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 993 struct virtio_hw *hw = vq->hw; 994 struct rte_mbuf *rxm; 995 uint16_t nb_used, num, nb_rx; 996 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 997 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 998 int error; 999 uint32_t i, nb_enqueued; 1000 uint32_t hdr_size; 1001 struct virtio_net_hdr *hdr; 1002 1003 nb_rx = 0; 1004 if (unlikely(hw->started == 0)) 1005 return nb_rx; 1006 1007 nb_used = virtqueue_nused(vq); 1008 1009 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1010 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1011 num = VIRTIO_MBUF_BURST_SZ; 1012 if (likely(num > DESC_PER_CACHELINE)) 1013 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1014 1015 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1016 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 1017 1018 nb_enqueued = 0; 1019 hdr_size = hw->vtnet_hdr_size; 1020 1021 for (i = 0; i < num ; i++) { 1022 rxm = rcv_pkts[i]; 1023 1024 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1025 1026 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1027 PMD_RX_LOG(ERR, "Packet drop"); 1028 nb_enqueued++; 1029 virtio_discard_rxbuf(vq, rxm); 1030 rxvq->stats.errors++; 1031 continue; 1032 } 1033 1034 rxm->port = rxvq->port_id; 1035 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1036 rxm->ol_flags = 0; 1037 rxm->vlan_tci = 0; 1038 1039 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1040 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1041 1042 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1043 RTE_PKTMBUF_HEADROOM - hdr_size); 1044 1045 if (hw->vlan_strip) 1046 rte_vlan_strip(rxm); 1047 1048 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1049 virtio_discard_rxbuf(vq, rxm); 1050 rxvq->stats.errors++; 1051 continue; 1052 } 1053 1054 virtio_rx_stats_updated(rxvq, rxm); 1055 1056 rx_pkts[nb_rx++] = rxm; 1057 } 1058 1059 rxvq->stats.packets += nb_rx; 1060 1061 /* Allocate new mbuf for the used descriptor */ 1062 if (likely(!virtqueue_full(vq))) { 1063 uint16_t free_cnt = vq->vq_free_cnt; 1064 struct rte_mbuf *new_pkts[free_cnt]; 1065 1066 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1067 free_cnt) == 0)) { 1068 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1069 free_cnt); 1070 if (unlikely(error)) { 1071 for (i = 0; i < free_cnt; i++) 1072 rte_pktmbuf_free(new_pkts[i]); 1073 } 1074 nb_enqueued += free_cnt; 1075 } else { 1076 struct rte_eth_dev *dev = 1077 &rte_eth_devices[rxvq->port_id]; 1078 dev->data->rx_mbuf_alloc_failed += free_cnt; 1079 } 1080 } 1081 1082 if (likely(nb_enqueued)) { 1083 vq_update_avail_idx(vq); 1084 1085 if (unlikely(virtqueue_kick_prepare(vq))) { 1086 virtqueue_notify(vq); 1087 PMD_RX_LOG(DEBUG, "Notified"); 1088 } 1089 } 1090 1091 return nb_rx; 1092 } 1093 1094 uint16_t 1095 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1096 uint16_t nb_pkts) 1097 { 1098 struct virtnet_rx *rxvq = rx_queue; 1099 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1100 struct virtio_hw *hw = vq->hw; 1101 struct rte_mbuf *rxm; 1102 uint16_t num, nb_rx; 1103 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1104 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1105 int error; 1106 uint32_t i, nb_enqueued; 1107 uint32_t hdr_size; 1108 struct virtio_net_hdr *hdr; 1109 1110 nb_rx = 0; 1111 if (unlikely(hw->started == 0)) 1112 return nb_rx; 1113 1114 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1115 if (likely(num > DESC_PER_CACHELINE)) 1116 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1117 1118 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1119 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1120 1121 nb_enqueued = 0; 1122 hdr_size = hw->vtnet_hdr_size; 1123 1124 for (i = 0; i < num; i++) { 1125 rxm = rcv_pkts[i]; 1126 1127 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1128 1129 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1130 PMD_RX_LOG(ERR, "Packet drop"); 1131 nb_enqueued++; 1132 virtio_discard_rxbuf(vq, rxm); 1133 rxvq->stats.errors++; 1134 continue; 1135 } 1136 1137 rxm->port = rxvq->port_id; 1138 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1139 rxm->ol_flags = 0; 1140 rxm->vlan_tci = 0; 1141 1142 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1143 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1144 1145 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1146 RTE_PKTMBUF_HEADROOM - hdr_size); 1147 1148 if (hw->vlan_strip) 1149 rte_vlan_strip(rxm); 1150 1151 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1152 virtio_discard_rxbuf(vq, rxm); 1153 rxvq->stats.errors++; 1154 continue; 1155 } 1156 1157 virtio_rx_stats_updated(rxvq, rxm); 1158 1159 rx_pkts[nb_rx++] = rxm; 1160 } 1161 1162 rxvq->stats.packets += nb_rx; 1163 1164 /* Allocate new mbuf for the used descriptor */ 1165 if (likely(!virtqueue_full(vq))) { 1166 uint16_t free_cnt = vq->vq_free_cnt; 1167 struct rte_mbuf *new_pkts[free_cnt]; 1168 1169 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1170 free_cnt) == 0)) { 1171 error = virtqueue_enqueue_recv_refill_packed(vq, 1172 new_pkts, free_cnt); 1173 if (unlikely(error)) { 1174 for (i = 0; i < free_cnt; i++) 1175 rte_pktmbuf_free(new_pkts[i]); 1176 } 1177 nb_enqueued += free_cnt; 1178 } else { 1179 struct rte_eth_dev *dev = 1180 &rte_eth_devices[rxvq->port_id]; 1181 dev->data->rx_mbuf_alloc_failed += free_cnt; 1182 } 1183 } 1184 1185 if (likely(nb_enqueued)) { 1186 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1187 virtqueue_notify(vq); 1188 PMD_RX_LOG(DEBUG, "Notified"); 1189 } 1190 } 1191 1192 return nb_rx; 1193 } 1194 1195 1196 uint16_t 1197 virtio_recv_pkts_inorder(void *rx_queue, 1198 struct rte_mbuf **rx_pkts, 1199 uint16_t nb_pkts) 1200 { 1201 struct virtnet_rx *rxvq = rx_queue; 1202 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1203 struct virtio_hw *hw = vq->hw; 1204 struct rte_mbuf *rxm; 1205 struct rte_mbuf *prev = NULL; 1206 uint16_t nb_used, num, nb_rx; 1207 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1208 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1209 int error; 1210 uint32_t nb_enqueued; 1211 uint32_t seg_num; 1212 uint32_t seg_res; 1213 uint32_t hdr_size; 1214 int32_t i; 1215 1216 nb_rx = 0; 1217 if (unlikely(hw->started == 0)) 1218 return nb_rx; 1219 1220 nb_used = virtqueue_nused(vq); 1221 nb_used = RTE_MIN(nb_used, nb_pkts); 1222 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1223 1224 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1225 1226 nb_enqueued = 0; 1227 seg_num = 1; 1228 seg_res = 0; 1229 hdr_size = hw->vtnet_hdr_size; 1230 1231 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1232 1233 for (i = 0; i < num; i++) { 1234 struct virtio_net_hdr_mrg_rxbuf *header; 1235 1236 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1237 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1238 1239 rxm = rcv_pkts[i]; 1240 1241 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1242 PMD_RX_LOG(ERR, "Packet drop"); 1243 nb_enqueued++; 1244 virtio_discard_rxbuf_inorder(vq, rxm); 1245 rxvq->stats.errors++; 1246 continue; 1247 } 1248 1249 header = (struct virtio_net_hdr_mrg_rxbuf *) 1250 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1251 - hdr_size); 1252 1253 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1254 seg_num = header->num_buffers; 1255 if (seg_num == 0) 1256 seg_num = 1; 1257 } else { 1258 seg_num = 1; 1259 } 1260 1261 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1262 rxm->nb_segs = seg_num; 1263 rxm->ol_flags = 0; 1264 rxm->vlan_tci = 0; 1265 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1266 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1267 1268 rxm->port = rxvq->port_id; 1269 1270 rx_pkts[nb_rx] = rxm; 1271 prev = rxm; 1272 1273 if (vq->hw->has_rx_offload && 1274 virtio_rx_offload(rxm, &header->hdr) < 0) { 1275 virtio_discard_rxbuf_inorder(vq, rxm); 1276 rxvq->stats.errors++; 1277 continue; 1278 } 1279 1280 if (hw->vlan_strip) 1281 rte_vlan_strip(rx_pkts[nb_rx]); 1282 1283 seg_res = seg_num - 1; 1284 1285 /* Merge remaining segments */ 1286 while (seg_res != 0 && i < (num - 1)) { 1287 i++; 1288 1289 rxm = rcv_pkts[i]; 1290 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1291 rxm->pkt_len = (uint32_t)(len[i]); 1292 rxm->data_len = (uint16_t)(len[i]); 1293 1294 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1295 1296 prev->next = rxm; 1297 prev = rxm; 1298 seg_res -= 1; 1299 } 1300 1301 if (!seg_res) { 1302 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1303 nb_rx++; 1304 } 1305 } 1306 1307 /* Last packet still need merge segments */ 1308 while (seg_res != 0) { 1309 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1310 VIRTIO_MBUF_BURST_SZ); 1311 1312 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1313 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1314 rcv_cnt); 1315 uint16_t extra_idx = 0; 1316 1317 rcv_cnt = num; 1318 while (extra_idx < rcv_cnt) { 1319 rxm = rcv_pkts[extra_idx]; 1320 rxm->data_off = 1321 RTE_PKTMBUF_HEADROOM - hdr_size; 1322 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1323 rxm->data_len = (uint16_t)(len[extra_idx]); 1324 prev->next = rxm; 1325 prev = rxm; 1326 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1327 extra_idx += 1; 1328 }; 1329 seg_res -= rcv_cnt; 1330 1331 if (!seg_res) { 1332 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1333 nb_rx++; 1334 } 1335 } else { 1336 PMD_RX_LOG(ERR, 1337 "No enough segments for packet."); 1338 rte_pktmbuf_free(rx_pkts[nb_rx]); 1339 rxvq->stats.errors++; 1340 break; 1341 } 1342 } 1343 1344 rxvq->stats.packets += nb_rx; 1345 1346 /* Allocate new mbuf for the used descriptor */ 1347 1348 if (likely(!virtqueue_full(vq))) { 1349 /* free_cnt may include mrg descs */ 1350 uint16_t free_cnt = vq->vq_free_cnt; 1351 struct rte_mbuf *new_pkts[free_cnt]; 1352 1353 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1354 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1355 free_cnt); 1356 if (unlikely(error)) { 1357 for (i = 0; i < free_cnt; i++) 1358 rte_pktmbuf_free(new_pkts[i]); 1359 } 1360 nb_enqueued += free_cnt; 1361 } else { 1362 struct rte_eth_dev *dev = 1363 &rte_eth_devices[rxvq->port_id]; 1364 dev->data->rx_mbuf_alloc_failed += free_cnt; 1365 } 1366 } 1367 1368 if (likely(nb_enqueued)) { 1369 vq_update_avail_idx(vq); 1370 1371 if (unlikely(virtqueue_kick_prepare(vq))) { 1372 virtqueue_notify(vq); 1373 PMD_RX_LOG(DEBUG, "Notified"); 1374 } 1375 } 1376 1377 return nb_rx; 1378 } 1379 1380 uint16_t 1381 virtio_recv_mergeable_pkts(void *rx_queue, 1382 struct rte_mbuf **rx_pkts, 1383 uint16_t nb_pkts) 1384 { 1385 struct virtnet_rx *rxvq = rx_queue; 1386 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1387 struct virtio_hw *hw = vq->hw; 1388 struct rte_mbuf *rxm; 1389 struct rte_mbuf *prev = NULL; 1390 uint16_t nb_used, num, nb_rx = 0; 1391 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1392 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1393 int error; 1394 uint32_t nb_enqueued = 0; 1395 uint32_t seg_num = 0; 1396 uint32_t seg_res = 0; 1397 uint32_t hdr_size = hw->vtnet_hdr_size; 1398 int32_t i; 1399 1400 if (unlikely(hw->started == 0)) 1401 return nb_rx; 1402 1403 nb_used = virtqueue_nused(vq); 1404 1405 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1406 1407 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1408 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1409 num = VIRTIO_MBUF_BURST_SZ; 1410 if (likely(num > DESC_PER_CACHELINE)) 1411 num = num - ((vq->vq_used_cons_idx + num) % 1412 DESC_PER_CACHELINE); 1413 1414 1415 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1416 1417 for (i = 0; i < num; i++) { 1418 struct virtio_net_hdr_mrg_rxbuf *header; 1419 1420 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1421 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1422 1423 rxm = rcv_pkts[i]; 1424 1425 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1426 PMD_RX_LOG(ERR, "Packet drop"); 1427 nb_enqueued++; 1428 virtio_discard_rxbuf(vq, rxm); 1429 rxvq->stats.errors++; 1430 continue; 1431 } 1432 1433 header = (struct virtio_net_hdr_mrg_rxbuf *) 1434 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1435 - hdr_size); 1436 seg_num = header->num_buffers; 1437 if (seg_num == 0) 1438 seg_num = 1; 1439 1440 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1441 rxm->nb_segs = seg_num; 1442 rxm->ol_flags = 0; 1443 rxm->vlan_tci = 0; 1444 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1445 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1446 1447 rxm->port = rxvq->port_id; 1448 1449 rx_pkts[nb_rx] = rxm; 1450 prev = rxm; 1451 1452 if (hw->has_rx_offload && 1453 virtio_rx_offload(rxm, &header->hdr) < 0) { 1454 virtio_discard_rxbuf(vq, rxm); 1455 rxvq->stats.errors++; 1456 continue; 1457 } 1458 1459 if (hw->vlan_strip) 1460 rte_vlan_strip(rx_pkts[nb_rx]); 1461 1462 seg_res = seg_num - 1; 1463 1464 /* Merge remaining segments */ 1465 while (seg_res != 0 && i < (num - 1)) { 1466 i++; 1467 1468 rxm = rcv_pkts[i]; 1469 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1470 rxm->pkt_len = (uint32_t)(len[i]); 1471 rxm->data_len = (uint16_t)(len[i]); 1472 1473 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1474 1475 prev->next = rxm; 1476 prev = rxm; 1477 seg_res -= 1; 1478 } 1479 1480 if (!seg_res) { 1481 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1482 nb_rx++; 1483 } 1484 } 1485 1486 /* Last packet still need merge segments */ 1487 while (seg_res != 0) { 1488 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1489 VIRTIO_MBUF_BURST_SZ); 1490 1491 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1492 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1493 rcv_cnt); 1494 uint16_t extra_idx = 0; 1495 1496 rcv_cnt = num; 1497 while (extra_idx < rcv_cnt) { 1498 rxm = rcv_pkts[extra_idx]; 1499 rxm->data_off = 1500 RTE_PKTMBUF_HEADROOM - hdr_size; 1501 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1502 rxm->data_len = (uint16_t)(len[extra_idx]); 1503 prev->next = rxm; 1504 prev = rxm; 1505 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1506 extra_idx += 1; 1507 }; 1508 seg_res -= rcv_cnt; 1509 1510 if (!seg_res) { 1511 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1512 nb_rx++; 1513 } 1514 } else { 1515 PMD_RX_LOG(ERR, 1516 "No enough segments for packet."); 1517 rte_pktmbuf_free(rx_pkts[nb_rx]); 1518 rxvq->stats.errors++; 1519 break; 1520 } 1521 } 1522 1523 rxvq->stats.packets += nb_rx; 1524 1525 /* Allocate new mbuf for the used descriptor */ 1526 if (likely(!virtqueue_full(vq))) { 1527 /* free_cnt may include mrg descs */ 1528 uint16_t free_cnt = vq->vq_free_cnt; 1529 struct rte_mbuf *new_pkts[free_cnt]; 1530 1531 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1532 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1533 free_cnt); 1534 if (unlikely(error)) { 1535 for (i = 0; i < free_cnt; i++) 1536 rte_pktmbuf_free(new_pkts[i]); 1537 } 1538 nb_enqueued += free_cnt; 1539 } else { 1540 struct rte_eth_dev *dev = 1541 &rte_eth_devices[rxvq->port_id]; 1542 dev->data->rx_mbuf_alloc_failed += free_cnt; 1543 } 1544 } 1545 1546 if (likely(nb_enqueued)) { 1547 vq_update_avail_idx(vq); 1548 1549 if (unlikely(virtqueue_kick_prepare(vq))) { 1550 virtqueue_notify(vq); 1551 PMD_RX_LOG(DEBUG, "Notified"); 1552 } 1553 } 1554 1555 return nb_rx; 1556 } 1557 1558 uint16_t 1559 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1560 struct rte_mbuf **rx_pkts, 1561 uint16_t nb_pkts) 1562 { 1563 struct virtnet_rx *rxvq = rx_queue; 1564 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1565 struct virtio_hw *hw = vq->hw; 1566 struct rte_mbuf *rxm; 1567 struct rte_mbuf *prev = NULL; 1568 uint16_t num, nb_rx = 0; 1569 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1570 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1571 uint32_t nb_enqueued = 0; 1572 uint32_t seg_num = 0; 1573 uint32_t seg_res = 0; 1574 uint32_t hdr_size = hw->vtnet_hdr_size; 1575 int32_t i; 1576 int error; 1577 1578 if (unlikely(hw->started == 0)) 1579 return nb_rx; 1580 1581 1582 num = nb_pkts; 1583 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1584 num = VIRTIO_MBUF_BURST_SZ; 1585 if (likely(num > DESC_PER_CACHELINE)) 1586 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1587 1588 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1589 1590 for (i = 0; i < num; i++) { 1591 struct virtio_net_hdr_mrg_rxbuf *header; 1592 1593 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1594 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1595 1596 rxm = rcv_pkts[i]; 1597 1598 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1599 PMD_RX_LOG(ERR, "Packet drop"); 1600 nb_enqueued++; 1601 virtio_discard_rxbuf(vq, rxm); 1602 rxvq->stats.errors++; 1603 continue; 1604 } 1605 1606 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1607 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1608 seg_num = header->num_buffers; 1609 1610 if (seg_num == 0) 1611 seg_num = 1; 1612 1613 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1614 rxm->nb_segs = seg_num; 1615 rxm->ol_flags = 0; 1616 rxm->vlan_tci = 0; 1617 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1618 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1619 1620 rxm->port = rxvq->port_id; 1621 rx_pkts[nb_rx] = rxm; 1622 prev = rxm; 1623 1624 if (hw->has_rx_offload && 1625 virtio_rx_offload(rxm, &header->hdr) < 0) { 1626 virtio_discard_rxbuf(vq, rxm); 1627 rxvq->stats.errors++; 1628 continue; 1629 } 1630 1631 if (hw->vlan_strip) 1632 rte_vlan_strip(rx_pkts[nb_rx]); 1633 1634 seg_res = seg_num - 1; 1635 1636 /* Merge remaining segments */ 1637 while (seg_res != 0 && i < (num - 1)) { 1638 i++; 1639 1640 rxm = rcv_pkts[i]; 1641 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1642 rxm->pkt_len = (uint32_t)(len[i]); 1643 rxm->data_len = (uint16_t)(len[i]); 1644 1645 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1646 1647 prev->next = rxm; 1648 prev = rxm; 1649 seg_res -= 1; 1650 } 1651 1652 if (!seg_res) { 1653 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1654 nb_rx++; 1655 } 1656 } 1657 1658 /* Last packet still need merge segments */ 1659 while (seg_res != 0) { 1660 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1661 VIRTIO_MBUF_BURST_SZ); 1662 uint16_t extra_idx = 0; 1663 1664 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1665 len, rcv_cnt); 1666 if (unlikely(rcv_cnt == 0)) { 1667 PMD_RX_LOG(ERR, "No enough segments for packet."); 1668 rte_pktmbuf_free(rx_pkts[nb_rx]); 1669 rxvq->stats.errors++; 1670 break; 1671 } 1672 1673 while (extra_idx < rcv_cnt) { 1674 rxm = rcv_pkts[extra_idx]; 1675 1676 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1677 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1678 rxm->data_len = (uint16_t)(len[extra_idx]); 1679 1680 prev->next = rxm; 1681 prev = rxm; 1682 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1683 extra_idx += 1; 1684 } 1685 seg_res -= rcv_cnt; 1686 if (!seg_res) { 1687 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1688 nb_rx++; 1689 } 1690 } 1691 1692 rxvq->stats.packets += nb_rx; 1693 1694 /* Allocate new mbuf for the used descriptor */ 1695 if (likely(!virtqueue_full(vq))) { 1696 /* free_cnt may include mrg descs */ 1697 uint16_t free_cnt = vq->vq_free_cnt; 1698 struct rte_mbuf *new_pkts[free_cnt]; 1699 1700 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1701 error = virtqueue_enqueue_recv_refill_packed(vq, 1702 new_pkts, free_cnt); 1703 if (unlikely(error)) { 1704 for (i = 0; i < free_cnt; i++) 1705 rte_pktmbuf_free(new_pkts[i]); 1706 } 1707 nb_enqueued += free_cnt; 1708 } else { 1709 struct rte_eth_dev *dev = 1710 &rte_eth_devices[rxvq->port_id]; 1711 dev->data->rx_mbuf_alloc_failed += free_cnt; 1712 } 1713 } 1714 1715 if (likely(nb_enqueued)) { 1716 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1717 virtqueue_notify(vq); 1718 PMD_RX_LOG(DEBUG, "Notified"); 1719 } 1720 } 1721 1722 return nb_rx; 1723 } 1724 1725 uint16_t 1726 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1727 uint16_t nb_pkts) 1728 { 1729 uint16_t nb_tx; 1730 int error; 1731 1732 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1733 struct rte_mbuf *m = tx_pkts[nb_tx]; 1734 1735 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1736 error = rte_validate_tx_offload(m); 1737 if (unlikely(error)) { 1738 rte_errno = -error; 1739 break; 1740 } 1741 #endif 1742 1743 /* Do VLAN tag insertion */ 1744 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) { 1745 error = rte_vlan_insert(&m); 1746 /* rte_vlan_insert() may change pointer 1747 * even in the case of failure 1748 */ 1749 tx_pkts[nb_tx] = m; 1750 1751 if (unlikely(error)) { 1752 rte_errno = -error; 1753 break; 1754 } 1755 } 1756 1757 error = rte_net_intel_cksum_prepare(m); 1758 if (unlikely(error)) { 1759 rte_errno = -error; 1760 break; 1761 } 1762 1763 if (m->ol_flags & PKT_TX_TCP_SEG) 1764 virtio_tso_fix_cksum(m); 1765 } 1766 1767 return nb_tx; 1768 } 1769 1770 uint16_t 1771 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 1772 uint16_t nb_pkts) 1773 { 1774 struct virtnet_tx *txvq = tx_queue; 1775 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1776 struct virtio_hw *hw = vq->hw; 1777 uint16_t hdr_size = hw->vtnet_hdr_size; 1778 uint16_t nb_tx = 0; 1779 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 1780 1781 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1782 return nb_tx; 1783 1784 if (unlikely(nb_pkts < 1)) 1785 return nb_pkts; 1786 1787 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1788 1789 if (nb_pkts > vq->vq_free_cnt) 1790 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 1791 in_order); 1792 1793 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1794 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1795 int can_push = 0, use_indirect = 0, slots, need; 1796 1797 /* optimize ring usage */ 1798 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1799 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1800 rte_mbuf_refcnt_read(txm) == 1 && 1801 RTE_MBUF_DIRECT(txm) && 1802 txm->nb_segs == 1 && 1803 rte_pktmbuf_headroom(txm) >= hdr_size && 1804 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1805 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1806 can_push = 1; 1807 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1808 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1809 use_indirect = 1; 1810 /* How many main ring entries are needed to this Tx? 1811 * indirect => 1 1812 * any_layout => number of segments 1813 * default => number of segments + 1 1814 */ 1815 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1816 need = slots - vq->vq_free_cnt; 1817 1818 /* Positive value indicates it need free vring descriptors */ 1819 if (unlikely(need > 0)) { 1820 virtio_xmit_cleanup_packed(vq, need, in_order); 1821 need = slots - vq->vq_free_cnt; 1822 if (unlikely(need > 0)) { 1823 PMD_TX_LOG(ERR, 1824 "No free tx descriptors to transmit"); 1825 break; 1826 } 1827 } 1828 1829 /* Enqueue Packet buffers */ 1830 if (can_push) 1831 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 1832 else 1833 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 1834 use_indirect, 0, 1835 in_order); 1836 1837 virtio_update_packet_stats(&txvq->stats, txm); 1838 } 1839 1840 txvq->stats.packets += nb_tx; 1841 1842 if (likely(nb_tx)) { 1843 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1844 virtqueue_notify(vq); 1845 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1846 } 1847 } 1848 1849 return nb_tx; 1850 } 1851 1852 uint16_t 1853 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1854 { 1855 struct virtnet_tx *txvq = tx_queue; 1856 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1857 struct virtio_hw *hw = vq->hw; 1858 uint16_t hdr_size = hw->vtnet_hdr_size; 1859 uint16_t nb_used, nb_tx = 0; 1860 1861 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1862 return nb_tx; 1863 1864 if (unlikely(nb_pkts < 1)) 1865 return nb_pkts; 1866 1867 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1868 1869 nb_used = virtqueue_nused(vq); 1870 1871 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1872 virtio_xmit_cleanup(vq, nb_used); 1873 1874 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1875 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1876 int can_push = 0, use_indirect = 0, slots, need; 1877 1878 /* optimize ring usage */ 1879 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1880 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1881 rte_mbuf_refcnt_read(txm) == 1 && 1882 RTE_MBUF_DIRECT(txm) && 1883 txm->nb_segs == 1 && 1884 rte_pktmbuf_headroom(txm) >= hdr_size && 1885 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1886 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1887 can_push = 1; 1888 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1889 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1890 use_indirect = 1; 1891 1892 /* How many main ring entries are needed to this Tx? 1893 * any_layout => number of segments 1894 * indirect => 1 1895 * default => number of segments + 1 1896 */ 1897 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1898 need = slots - vq->vq_free_cnt; 1899 1900 /* Positive value indicates it need free vring descriptors */ 1901 if (unlikely(need > 0)) { 1902 nb_used = virtqueue_nused(vq); 1903 1904 need = RTE_MIN(need, (int)nb_used); 1905 1906 virtio_xmit_cleanup(vq, need); 1907 need = slots - vq->vq_free_cnt; 1908 if (unlikely(need > 0)) { 1909 PMD_TX_LOG(ERR, 1910 "No free tx descriptors to transmit"); 1911 break; 1912 } 1913 } 1914 1915 /* Enqueue Packet buffers */ 1916 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 1917 can_push, 0); 1918 1919 virtio_update_packet_stats(&txvq->stats, txm); 1920 } 1921 1922 txvq->stats.packets += nb_tx; 1923 1924 if (likely(nb_tx)) { 1925 vq_update_avail_idx(vq); 1926 1927 if (unlikely(virtqueue_kick_prepare(vq))) { 1928 virtqueue_notify(vq); 1929 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1930 } 1931 } 1932 1933 return nb_tx; 1934 } 1935 1936 static __rte_always_inline int 1937 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 1938 { 1939 uint16_t nb_used, nb_clean, nb_descs; 1940 1941 nb_descs = vq->vq_free_cnt + need; 1942 nb_used = virtqueue_nused(vq); 1943 nb_clean = RTE_MIN(need, (int)nb_used); 1944 1945 virtio_xmit_cleanup_inorder(vq, nb_clean); 1946 1947 return nb_descs - vq->vq_free_cnt; 1948 } 1949 1950 uint16_t 1951 virtio_xmit_pkts_inorder(void *tx_queue, 1952 struct rte_mbuf **tx_pkts, 1953 uint16_t nb_pkts) 1954 { 1955 struct virtnet_tx *txvq = tx_queue; 1956 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1957 struct virtio_hw *hw = vq->hw; 1958 uint16_t hdr_size = hw->vtnet_hdr_size; 1959 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 1960 struct rte_mbuf *inorder_pkts[nb_pkts]; 1961 int need; 1962 1963 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1964 return nb_tx; 1965 1966 if (unlikely(nb_pkts < 1)) 1967 return nb_pkts; 1968 1969 VIRTQUEUE_DUMP(vq); 1970 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1971 nb_used = virtqueue_nused(vq); 1972 1973 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1974 virtio_xmit_cleanup_inorder(vq, nb_used); 1975 1976 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1977 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1978 int slots; 1979 1980 /* optimize ring usage */ 1981 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1982 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1983 rte_mbuf_refcnt_read(txm) == 1 && 1984 RTE_MBUF_DIRECT(txm) && 1985 txm->nb_segs == 1 && 1986 rte_pktmbuf_headroom(txm) >= hdr_size && 1987 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1988 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) { 1989 inorder_pkts[nb_inorder_pkts] = txm; 1990 nb_inorder_pkts++; 1991 1992 continue; 1993 } 1994 1995 if (nb_inorder_pkts) { 1996 need = nb_inorder_pkts - vq->vq_free_cnt; 1997 if (unlikely(need > 0)) { 1998 need = virtio_xmit_try_cleanup_inorder(vq, 1999 need); 2000 if (unlikely(need > 0)) { 2001 PMD_TX_LOG(ERR, 2002 "No free tx descriptors to " 2003 "transmit"); 2004 break; 2005 } 2006 } 2007 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2008 nb_inorder_pkts); 2009 nb_inorder_pkts = 0; 2010 } 2011 2012 slots = txm->nb_segs + 1; 2013 need = slots - vq->vq_free_cnt; 2014 if (unlikely(need > 0)) { 2015 need = virtio_xmit_try_cleanup_inorder(vq, slots); 2016 2017 if (unlikely(need > 0)) { 2018 PMD_TX_LOG(ERR, 2019 "No free tx descriptors to transmit"); 2020 break; 2021 } 2022 } 2023 /* Enqueue Packet buffers */ 2024 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 2025 2026 virtio_update_packet_stats(&txvq->stats, txm); 2027 } 2028 2029 /* Transmit all inorder packets */ 2030 if (nb_inorder_pkts) { 2031 need = nb_inorder_pkts - vq->vq_free_cnt; 2032 if (unlikely(need > 0)) { 2033 need = virtio_xmit_try_cleanup_inorder(vq, 2034 need); 2035 if (unlikely(need > 0)) { 2036 PMD_TX_LOG(ERR, 2037 "No free tx descriptors to transmit"); 2038 nb_inorder_pkts = vq->vq_free_cnt; 2039 nb_tx -= need; 2040 } 2041 } 2042 2043 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2044 nb_inorder_pkts); 2045 } 2046 2047 txvq->stats.packets += nb_tx; 2048 2049 if (likely(nb_tx)) { 2050 vq_update_avail_idx(vq); 2051 2052 if (unlikely(virtqueue_kick_prepare(vq))) { 2053 virtqueue_notify(vq); 2054 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2055 } 2056 } 2057 2058 VIRTQUEUE_DUMP(vq); 2059 2060 return nb_tx; 2061 } 2062 2063 __rte_weak uint16_t 2064 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, 2065 struct rte_mbuf **rx_pkts __rte_unused, 2066 uint16_t nb_pkts __rte_unused) 2067 { 2068 return 0; 2069 } 2070 2071 __rte_weak uint16_t 2072 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, 2073 struct rte_mbuf **tx_pkts __rte_unused, 2074 uint16_t nb_pkts __rte_unused) 2075 { 2076 return 0; 2077 } 2078