1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_cycles.h> 12 #include <rte_memory.h> 13 #include <rte_branch_prediction.h> 14 #include <rte_mempool.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_ether.h> 18 #include <ethdev_driver.h> 19 #include <rte_prefetch.h> 20 #include <rte_string_fns.h> 21 #include <rte_errno.h> 22 #include <rte_byteorder.h> 23 #include <rte_net.h> 24 #include <rte_ip.h> 25 #include <rte_udp.h> 26 #include <rte_tcp.h> 27 28 #include "virtio_logs.h" 29 #include "virtio_ethdev.h" 30 #include "virtio.h" 31 #include "virtqueue.h" 32 #include "virtio_rxtx.h" 33 #include "virtio_rxtx_simple.h" 34 #include "virtio_ring.h" 35 36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 38 #else 39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 40 #endif 41 42 int 43 virtio_dev_rx_queue_done(void *rxq, uint16_t offset) 44 { 45 struct virtnet_rx *rxvq = rxq; 46 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 47 48 return virtqueue_nused(vq) >= offset; 49 } 50 51 void 52 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 53 { 54 vq->vq_free_cnt += num; 55 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 56 } 57 58 void 59 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 60 { 61 struct vring_desc *dp, *dp_tail; 62 struct vq_desc_extra *dxp; 63 uint16_t desc_idx_last = desc_idx; 64 65 dp = &vq->vq_split.ring.desc[desc_idx]; 66 dxp = &vq->vq_descx[desc_idx]; 67 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 68 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 69 while (dp->flags & VRING_DESC_F_NEXT) { 70 desc_idx_last = dp->next; 71 dp = &vq->vq_split.ring.desc[dp->next]; 72 } 73 } 74 dxp->ndescs = 0; 75 76 /* 77 * We must append the existing free chain, if any, to the end of 78 * newly freed chain. If the virtqueue was completely used, then 79 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 80 */ 81 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 82 vq->vq_desc_head_idx = desc_idx; 83 } else { 84 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 85 dp_tail->next = desc_idx; 86 } 87 88 vq->vq_desc_tail_idx = desc_idx_last; 89 dp->next = VQ_RING_DESC_CHAIN_END; 90 } 91 92 void 93 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) 94 { 95 uint32_t s = mbuf->pkt_len; 96 struct rte_ether_addr *ea; 97 98 stats->bytes += s; 99 100 if (s == 64) { 101 stats->size_bins[1]++; 102 } else if (s > 64 && s < 1024) { 103 uint32_t bin; 104 105 /* count zeros, and offset into correct bin */ 106 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 107 stats->size_bins[bin]++; 108 } else { 109 if (s < 64) 110 stats->size_bins[0]++; 111 else if (s < 1519) 112 stats->size_bins[6]++; 113 else 114 stats->size_bins[7]++; 115 } 116 117 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 118 if (rte_is_multicast_ether_addr(ea)) { 119 if (rte_is_broadcast_ether_addr(ea)) 120 stats->broadcast++; 121 else 122 stats->multicast++; 123 } 124 } 125 126 static inline void 127 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 128 { 129 VIRTIO_DUMP_PACKET(m, m->data_len); 130 131 virtio_update_packet_stats(&rxvq->stats, m); 132 } 133 134 static uint16_t 135 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 136 struct rte_mbuf **rx_pkts, 137 uint32_t *len, 138 uint16_t num) 139 { 140 struct rte_mbuf *cookie; 141 uint16_t used_idx; 142 uint16_t id; 143 struct vring_packed_desc *desc; 144 uint16_t i; 145 146 desc = vq->vq_packed.ring.desc; 147 148 for (i = 0; i < num; i++) { 149 used_idx = vq->vq_used_cons_idx; 150 /* desc_is_used has a load-acquire or rte_io_rmb inside 151 * and wait for used desc in virtqueue. 152 */ 153 if (!desc_is_used(&desc[used_idx], vq)) 154 return i; 155 len[i] = desc[used_idx].len; 156 id = desc[used_idx].id; 157 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 158 if (unlikely(cookie == NULL)) { 159 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 160 vq->vq_used_cons_idx); 161 break; 162 } 163 rte_prefetch0(cookie); 164 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 165 rx_pkts[i] = cookie; 166 167 vq->vq_free_cnt++; 168 vq->vq_used_cons_idx++; 169 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 170 vq->vq_used_cons_idx -= vq->vq_nentries; 171 vq->vq_packed.used_wrap_counter ^= 1; 172 } 173 } 174 175 return i; 176 } 177 178 static uint16_t 179 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 180 uint32_t *len, uint16_t num) 181 { 182 struct vring_used_elem *uep; 183 struct rte_mbuf *cookie; 184 uint16_t used_idx, desc_idx; 185 uint16_t i; 186 187 /* Caller does the check */ 188 for (i = 0; i < num ; i++) { 189 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 190 uep = &vq->vq_split.ring.used->ring[used_idx]; 191 desc_idx = (uint16_t) uep->id; 192 len[i] = uep->len; 193 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 194 195 if (unlikely(cookie == NULL)) { 196 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 197 vq->vq_used_cons_idx); 198 break; 199 } 200 201 rte_prefetch0(cookie); 202 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 203 rx_pkts[i] = cookie; 204 vq->vq_used_cons_idx++; 205 vq_ring_free_chain(vq, desc_idx); 206 vq->vq_descx[desc_idx].cookie = NULL; 207 } 208 209 return i; 210 } 211 212 static uint16_t 213 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 214 struct rte_mbuf **rx_pkts, 215 uint32_t *len, 216 uint16_t num) 217 { 218 struct vring_used_elem *uep; 219 struct rte_mbuf *cookie; 220 uint16_t used_idx = 0; 221 uint16_t i; 222 223 if (unlikely(num == 0)) 224 return 0; 225 226 for (i = 0; i < num; i++) { 227 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 228 /* Desc idx same as used idx */ 229 uep = &vq->vq_split.ring.used->ring[used_idx]; 230 len[i] = uep->len; 231 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 232 233 if (unlikely(cookie == NULL)) { 234 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 235 vq->vq_used_cons_idx); 236 break; 237 } 238 239 rte_prefetch0(cookie); 240 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 241 rx_pkts[i] = cookie; 242 vq->vq_used_cons_idx++; 243 vq->vq_descx[used_idx].cookie = NULL; 244 } 245 246 vq_ring_free_inorder(vq, used_idx, i); 247 return i; 248 } 249 250 static inline int 251 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 252 struct rte_mbuf **cookies, 253 uint16_t num) 254 { 255 struct vq_desc_extra *dxp; 256 struct virtio_hw *hw = vq->hw; 257 struct vring_desc *start_dp; 258 uint16_t head_idx, idx, i = 0; 259 260 if (unlikely(vq->vq_free_cnt == 0)) 261 return -ENOSPC; 262 if (unlikely(vq->vq_free_cnt < num)) 263 return -EMSGSIZE; 264 265 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 266 start_dp = vq->vq_split.ring.desc; 267 268 while (i < num) { 269 idx = head_idx & (vq->vq_nentries - 1); 270 dxp = &vq->vq_descx[idx]; 271 dxp->cookie = (void *)cookies[i]; 272 dxp->ndescs = 1; 273 274 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) + 275 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 276 start_dp[idx].len = cookies[i]->buf_len - 277 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 278 start_dp[idx].flags = VRING_DESC_F_WRITE; 279 280 vq_update_avail_ring(vq, idx); 281 head_idx++; 282 i++; 283 } 284 285 vq->vq_desc_head_idx += num; 286 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 287 return 0; 288 } 289 290 static inline int 291 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 292 uint16_t num) 293 { 294 struct vq_desc_extra *dxp; 295 struct virtio_hw *hw = vq->hw; 296 struct vring_desc *start_dp = vq->vq_split.ring.desc; 297 uint16_t idx, i; 298 299 if (unlikely(vq->vq_free_cnt == 0)) 300 return -ENOSPC; 301 if (unlikely(vq->vq_free_cnt < num)) 302 return -EMSGSIZE; 303 304 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 305 return -EFAULT; 306 307 for (i = 0; i < num; i++) { 308 idx = vq->vq_desc_head_idx; 309 dxp = &vq->vq_descx[idx]; 310 dxp->cookie = (void *)cookie[i]; 311 dxp->ndescs = 1; 312 313 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + 314 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 315 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + 316 hw->vtnet_hdr_size; 317 start_dp[idx].flags = VRING_DESC_F_WRITE; 318 vq->vq_desc_head_idx = start_dp[idx].next; 319 vq_update_avail_ring(vq, idx); 320 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 321 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 322 break; 323 } 324 } 325 326 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 327 328 return 0; 329 } 330 331 static inline void 332 virtqueue_refill_single_packed(struct virtqueue *vq, 333 struct vring_packed_desc *dp, 334 struct rte_mbuf *cookie) 335 { 336 uint16_t flags = vq->vq_packed.cached_flags; 337 struct virtio_hw *hw = vq->hw; 338 339 dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 340 dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 341 342 virtqueue_store_flags_packed(dp, flags, hw->weak_barriers); 343 344 if (++vq->vq_avail_idx >= vq->vq_nentries) { 345 vq->vq_avail_idx -= vq->vq_nentries; 346 vq->vq_packed.cached_flags ^= 347 VRING_PACKED_DESC_F_AVAIL_USED; 348 flags = vq->vq_packed.cached_flags; 349 } 350 } 351 352 static inline int 353 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq, 354 struct rte_mbuf **cookie, uint16_t num) 355 { 356 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 357 struct vq_desc_extra *dxp; 358 uint16_t idx; 359 int i; 360 361 if (unlikely(vq->vq_free_cnt == 0)) 362 return -ENOSPC; 363 if (unlikely(vq->vq_free_cnt < num)) 364 return -EMSGSIZE; 365 366 for (i = 0; i < num; i++) { 367 idx = vq->vq_avail_idx; 368 dxp = &vq->vq_descx[idx]; 369 dxp->cookie = (void *)cookie[i]; 370 dxp->ndescs = 1; 371 372 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 373 } 374 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 375 return 0; 376 } 377 378 static inline int 379 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 380 struct rte_mbuf **cookie, uint16_t num) 381 { 382 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 383 struct vq_desc_extra *dxp; 384 uint16_t idx, did; 385 int i; 386 387 if (unlikely(vq->vq_free_cnt == 0)) 388 return -ENOSPC; 389 if (unlikely(vq->vq_free_cnt < num)) 390 return -EMSGSIZE; 391 392 for (i = 0; i < num; i++) { 393 idx = vq->vq_avail_idx; 394 did = start_dp[idx].id; 395 dxp = &vq->vq_descx[did]; 396 dxp->cookie = (void *)cookie[i]; 397 dxp->ndescs = 1; 398 399 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 400 } 401 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 402 return 0; 403 } 404 405 /* When doing TSO, the IP length is not included in the pseudo header 406 * checksum of the packet given to the PMD, but for virtio it is 407 * expected. 408 */ 409 static void 410 virtio_tso_fix_cksum(struct rte_mbuf *m) 411 { 412 /* common case: header is not fragmented */ 413 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 414 m->l4_len)) { 415 struct rte_ipv4_hdr *iph; 416 struct rte_ipv6_hdr *ip6h; 417 struct rte_tcp_hdr *th; 418 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; 419 uint32_t tmp; 420 421 iph = rte_pktmbuf_mtod_offset(m, 422 struct rte_ipv4_hdr *, m->l2_len); 423 th = RTE_PTR_ADD(iph, m->l3_len); 424 if ((iph->version_ihl >> 4) == 4) { 425 iph->hdr_checksum = 0; 426 iph->hdr_checksum = rte_ipv4_cksum(iph); 427 ip_len = iph->total_length; 428 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - 429 m->l3_len); 430 } else { 431 ip6h = (struct rte_ipv6_hdr *)iph; 432 ip_paylen = ip6h->payload_len; 433 } 434 435 /* calculate the new phdr checksum not including ip_paylen */ 436 prev_cksum = th->cksum; 437 tmp = prev_cksum; 438 tmp += ip_paylen; 439 tmp = (tmp & 0xffff) + (tmp >> 16); 440 new_cksum = tmp; 441 442 /* replace it in the packet */ 443 th->cksum = new_cksum; 444 } 445 } 446 447 448 449 450 static inline void 451 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 452 struct rte_mbuf **cookies, 453 uint16_t num) 454 { 455 struct vq_desc_extra *dxp; 456 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 457 struct vring_desc *start_dp; 458 struct virtio_net_hdr *hdr; 459 uint16_t idx; 460 int16_t head_size = vq->hw->vtnet_hdr_size; 461 uint16_t i = 0; 462 463 idx = vq->vq_desc_head_idx; 464 start_dp = vq->vq_split.ring.desc; 465 466 while (i < num) { 467 idx = idx & (vq->vq_nentries - 1); 468 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 469 dxp->cookie = (void *)cookies[i]; 470 dxp->ndescs = 1; 471 virtio_update_packet_stats(&txvq->stats, cookies[i]); 472 473 hdr = rte_pktmbuf_mtod_offset(cookies[i], 474 struct virtio_net_hdr *, -head_size); 475 476 /* if offload disabled, hdr is not zeroed yet, do it now */ 477 if (!vq->hw->has_tx_offload) 478 virtqueue_clear_net_hdr(hdr); 479 else 480 virtqueue_xmit_offload(hdr, cookies[i]); 481 482 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size; 483 start_dp[idx].len = cookies[i]->data_len + head_size; 484 start_dp[idx].flags = 0; 485 486 487 vq_update_avail_ring(vq, idx); 488 489 idx++; 490 i++; 491 }; 492 493 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 494 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 495 } 496 497 static inline void 498 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 499 struct rte_mbuf *cookie, 500 int in_order) 501 { 502 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 503 struct vring_packed_desc *dp; 504 struct vq_desc_extra *dxp; 505 uint16_t idx, id, flags; 506 int16_t head_size = vq->hw->vtnet_hdr_size; 507 struct virtio_net_hdr *hdr; 508 509 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 510 idx = vq->vq_avail_idx; 511 dp = &vq->vq_packed.ring.desc[idx]; 512 513 dxp = &vq->vq_descx[id]; 514 dxp->ndescs = 1; 515 dxp->cookie = cookie; 516 517 flags = vq->vq_packed.cached_flags; 518 519 /* prepend cannot fail, checked by caller */ 520 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 521 -head_size); 522 523 /* if offload disabled, hdr is not zeroed yet, do it now */ 524 if (!vq->hw->has_tx_offload) 525 virtqueue_clear_net_hdr(hdr); 526 else 527 virtqueue_xmit_offload(hdr, cookie); 528 529 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; 530 dp->len = cookie->data_len + head_size; 531 dp->id = id; 532 533 if (++vq->vq_avail_idx >= vq->vq_nentries) { 534 vq->vq_avail_idx -= vq->vq_nentries; 535 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 536 } 537 538 vq->vq_free_cnt--; 539 540 if (!in_order) { 541 vq->vq_desc_head_idx = dxp->next; 542 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 543 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 544 } 545 546 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 547 } 548 549 static inline void 550 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 551 uint16_t needed, int use_indirect, int can_push, 552 int in_order) 553 { 554 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 555 struct vq_desc_extra *dxp; 556 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 557 struct vring_desc *start_dp; 558 uint16_t seg_num = cookie->nb_segs; 559 uint16_t head_idx, idx; 560 int16_t head_size = vq->hw->vtnet_hdr_size; 561 bool prepend_header = false; 562 struct virtio_net_hdr *hdr; 563 564 head_idx = vq->vq_desc_head_idx; 565 idx = head_idx; 566 if (in_order) 567 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 568 else 569 dxp = &vq->vq_descx[idx]; 570 dxp->cookie = (void *)cookie; 571 dxp->ndescs = needed; 572 573 start_dp = vq->vq_split.ring.desc; 574 575 if (can_push) { 576 /* prepend cannot fail, checked by caller */ 577 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 578 -head_size); 579 prepend_header = true; 580 581 /* if offload disabled, it is not zeroed below, do it now */ 582 if (!vq->hw->has_tx_offload) 583 virtqueue_clear_net_hdr(hdr); 584 } else if (use_indirect) { 585 /* setup tx ring slot to point to indirect 586 * descriptor list stored in reserved region. 587 * 588 * the first slot in indirect ring is already preset 589 * to point to the header in reserved region 590 */ 591 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 592 RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 593 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 594 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 595 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 596 597 /* loop below will fill in rest of the indirect elements */ 598 start_dp = txr[idx].tx_indir; 599 idx = 1; 600 } else { 601 /* setup first tx ring slot to point to header 602 * stored in reserved region. 603 */ 604 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 605 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 606 start_dp[idx].len = vq->hw->vtnet_hdr_size; 607 start_dp[idx].flags = VRING_DESC_F_NEXT; 608 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 609 610 idx = start_dp[idx].next; 611 } 612 613 if (vq->hw->has_tx_offload) 614 virtqueue_xmit_offload(hdr, cookie); 615 616 do { 617 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 618 start_dp[idx].len = cookie->data_len; 619 if (prepend_header) { 620 start_dp[idx].addr -= head_size; 621 start_dp[idx].len += head_size; 622 prepend_header = false; 623 } 624 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 625 idx = start_dp[idx].next; 626 } while ((cookie = cookie->next) != NULL); 627 628 if (use_indirect) 629 idx = vq->vq_split.ring.desc[head_idx].next; 630 631 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 632 633 vq->vq_desc_head_idx = idx; 634 vq_update_avail_ring(vq, head_idx); 635 636 if (!in_order) { 637 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 638 vq->vq_desc_tail_idx = idx; 639 } 640 } 641 642 void 643 virtio_dev_cq_start(struct rte_eth_dev *dev) 644 { 645 struct virtio_hw *hw = dev->data->dev_private; 646 647 if (hw->cvq) { 648 rte_spinlock_init(&hw->cvq->lock); 649 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); 650 } 651 } 652 653 int 654 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 655 uint16_t queue_idx, 656 uint16_t nb_desc, 657 unsigned int socket_id __rte_unused, 658 const struct rte_eth_rxconf *rx_conf, 659 struct rte_mempool *mp) 660 { 661 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 662 struct virtio_hw *hw = dev->data->dev_private; 663 struct virtqueue *vq = hw->vqs[vq_idx]; 664 struct virtnet_rx *rxvq; 665 uint16_t rx_free_thresh; 666 uint16_t buf_size; 667 const char *error; 668 669 PMD_INIT_FUNC_TRACE(); 670 671 if (rx_conf->rx_deferred_start) { 672 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 673 return -EINVAL; 674 } 675 676 buf_size = virtio_rx_mem_pool_buf_size(mp); 677 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size, 678 hw->rx_ol_scatter, &error)) { 679 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s", 680 queue_idx, error); 681 return -EINVAL; 682 } 683 684 rx_free_thresh = rx_conf->rx_free_thresh; 685 if (rx_free_thresh == 0) 686 rx_free_thresh = 687 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); 688 689 if (rx_free_thresh & 0x3) { 690 PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four." 691 " (rx_free_thresh=%u port=%u queue=%u)", 692 rx_free_thresh, dev->data->port_id, queue_idx); 693 return -EINVAL; 694 } 695 696 if (rx_free_thresh >= vq->vq_nentries) { 697 PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the " 698 "number of RX entries (%u)." 699 " (rx_free_thresh=%u port=%u queue=%u)", 700 vq->vq_nentries, 701 rx_free_thresh, dev->data->port_id, queue_idx); 702 return -EINVAL; 703 } 704 vq->vq_free_thresh = rx_free_thresh; 705 706 /* 707 * For split ring vectorized path descriptors number must be 708 * equal to the ring size. 709 */ 710 if (nb_desc > vq->vq_nentries || 711 (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) { 712 nb_desc = vq->vq_nentries; 713 } 714 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 715 716 rxvq = &vq->rxq; 717 rxvq->queue_id = queue_idx; 718 rxvq->mpool = mp; 719 dev->data->rx_queues[queue_idx] = rxvq; 720 721 return 0; 722 } 723 724 int 725 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 726 { 727 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 728 struct virtio_hw *hw = dev->data->dev_private; 729 struct virtqueue *vq = hw->vqs[vq_idx]; 730 struct virtnet_rx *rxvq = &vq->rxq; 731 struct rte_mbuf *m; 732 uint16_t desc_idx; 733 int error, nbufs, i; 734 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 735 736 PMD_INIT_FUNC_TRACE(); 737 738 /* Allocate blank mbufs for the each rx descriptor */ 739 nbufs = 0; 740 741 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 742 for (desc_idx = 0; desc_idx < vq->vq_nentries; 743 desc_idx++) { 744 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 745 vq->vq_split.ring.desc[desc_idx].flags = 746 VRING_DESC_F_WRITE; 747 } 748 749 virtio_rxq_vec_setup(rxvq); 750 } 751 752 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); 753 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) 754 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; 755 756 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 757 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 758 virtio_rxq_rearm_vec(rxvq); 759 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 760 } 761 } else if (!virtio_with_packed_queue(vq->hw) && in_order) { 762 if ((!virtqueue_full(vq))) { 763 uint16_t free_cnt = vq->vq_free_cnt; 764 struct rte_mbuf *pkts[free_cnt]; 765 766 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 767 free_cnt)) { 768 error = virtqueue_enqueue_refill_inorder(vq, 769 pkts, 770 free_cnt); 771 if (unlikely(error)) { 772 for (i = 0; i < free_cnt; i++) 773 rte_pktmbuf_free(pkts[i]); 774 } else { 775 nbufs += free_cnt; 776 } 777 } 778 779 vq_update_avail_idx(vq); 780 } 781 } else { 782 while (!virtqueue_full(vq)) { 783 m = rte_mbuf_raw_alloc(rxvq->mpool); 784 if (m == NULL) 785 break; 786 787 /* Enqueue allocated buffers */ 788 if (virtio_with_packed_queue(vq->hw)) 789 error = virtqueue_enqueue_recv_refill_packed_init(vq, 790 &m, 1); 791 else 792 error = virtqueue_enqueue_recv_refill(vq, 793 &m, 1); 794 if (error) { 795 rte_pktmbuf_free(m); 796 break; 797 } 798 nbufs++; 799 } 800 801 if (!virtio_with_packed_queue(vq->hw)) 802 vq_update_avail_idx(vq); 803 } 804 805 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); 806 807 VIRTQUEUE_DUMP(vq); 808 809 return 0; 810 } 811 812 /* 813 * struct rte_eth_dev *dev: Used to update dev 814 * uint16_t nb_desc: Defaults to values read from config space 815 * unsigned int socket_id: Used to allocate memzone 816 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 817 * uint16_t queue_idx: Just used as an index in dev txq list 818 */ 819 int 820 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 821 uint16_t queue_idx, 822 uint16_t nb_desc, 823 unsigned int socket_id __rte_unused, 824 const struct rte_eth_txconf *tx_conf) 825 { 826 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 827 struct virtio_hw *hw = dev->data->dev_private; 828 struct virtqueue *vq = hw->vqs[vq_idx]; 829 struct virtnet_tx *txvq; 830 uint16_t tx_free_thresh; 831 832 PMD_INIT_FUNC_TRACE(); 833 834 if (tx_conf->tx_deferred_start) { 835 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 836 return -EINVAL; 837 } 838 839 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 840 nb_desc = vq->vq_nentries; 841 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 842 843 txvq = &vq->txq; 844 txvq->queue_id = queue_idx; 845 846 tx_free_thresh = tx_conf->tx_free_thresh; 847 if (tx_free_thresh == 0) 848 tx_free_thresh = 849 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 850 851 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 852 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " 853 "number of TX entries minus 3 (%u)." 854 " (tx_free_thresh=%u port=%u queue=%u)", 855 vq->vq_nentries - 3, 856 tx_free_thresh, dev->data->port_id, queue_idx); 857 return -EINVAL; 858 } 859 860 vq->vq_free_thresh = tx_free_thresh; 861 862 dev->data->tx_queues[queue_idx] = txvq; 863 return 0; 864 } 865 866 int 867 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 868 uint16_t queue_idx) 869 { 870 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 871 struct virtio_hw *hw = dev->data->dev_private; 872 struct virtqueue *vq = hw->vqs[vq_idx]; 873 874 PMD_INIT_FUNC_TRACE(); 875 876 if (!virtio_with_packed_queue(hw)) { 877 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 878 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 879 } 880 881 VIRTQUEUE_DUMP(vq); 882 883 return 0; 884 } 885 886 static inline void 887 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 888 { 889 int error; 890 /* 891 * Requeue the discarded mbuf. This should always be 892 * successful since it was just dequeued. 893 */ 894 if (virtio_with_packed_queue(vq->hw)) 895 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 896 else 897 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 898 899 if (unlikely(error)) { 900 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 901 rte_pktmbuf_free(m); 902 } 903 } 904 905 static inline void 906 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 907 { 908 int error; 909 910 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 911 if (unlikely(error)) { 912 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 913 rte_pktmbuf_free(m); 914 } 915 } 916 917 /* Optionally fill offload information in structure */ 918 static inline int 919 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 920 { 921 struct rte_net_hdr_lens hdr_lens; 922 uint32_t hdrlen, ptype; 923 int l4_supported = 0; 924 925 /* nothing to do */ 926 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 927 return 0; 928 929 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; 930 931 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 932 m->packet_type = ptype; 933 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 934 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 935 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 936 l4_supported = 1; 937 938 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 939 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 940 if (hdr->csum_start <= hdrlen && l4_supported) { 941 m->ol_flags |= PKT_RX_L4_CKSUM_NONE; 942 } else { 943 /* Unknown proto or tunnel, do sw cksum. We can assume 944 * the cksum field is in the first segment since the 945 * buffers we provided to the host are large enough. 946 * In case of SCTP, this will be wrong since it's a CRC 947 * but there's nothing we can do. 948 */ 949 uint16_t csum = 0, off; 950 951 if (rte_raw_cksum_mbuf(m, hdr->csum_start, 952 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 953 &csum) < 0) 954 return -EINVAL; 955 if (likely(csum != 0xffff)) 956 csum = ~csum; 957 off = hdr->csum_offset + hdr->csum_start; 958 if (rte_pktmbuf_data_len(m) >= off + 1) 959 *rte_pktmbuf_mtod_offset(m, uint16_t *, 960 off) = csum; 961 } 962 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 963 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 964 } 965 966 /* GSO request, save required information in mbuf */ 967 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 968 /* Check unsupported modes */ 969 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 970 (hdr->gso_size == 0)) { 971 return -EINVAL; 972 } 973 974 /* Update mss lengthes in mbuf */ 975 m->tso_segsz = hdr->gso_size; 976 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 977 case VIRTIO_NET_HDR_GSO_TCPV4: 978 case VIRTIO_NET_HDR_GSO_TCPV6: 979 m->ol_flags |= PKT_RX_LRO | \ 980 PKT_RX_L4_CKSUM_NONE; 981 break; 982 default: 983 return -EINVAL; 984 } 985 } 986 987 return 0; 988 } 989 990 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 991 uint16_t 992 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 993 { 994 struct virtnet_rx *rxvq = rx_queue; 995 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 996 struct virtio_hw *hw = vq->hw; 997 struct rte_mbuf *rxm; 998 uint16_t nb_used, num, nb_rx; 999 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1000 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1001 int error; 1002 uint32_t i, nb_enqueued; 1003 uint32_t hdr_size; 1004 struct virtio_net_hdr *hdr; 1005 1006 nb_rx = 0; 1007 if (unlikely(hw->started == 0)) 1008 return nb_rx; 1009 1010 nb_used = virtqueue_nused(vq); 1011 1012 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1013 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1014 num = VIRTIO_MBUF_BURST_SZ; 1015 if (likely(num > DESC_PER_CACHELINE)) 1016 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1017 1018 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1019 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 1020 1021 nb_enqueued = 0; 1022 hdr_size = hw->vtnet_hdr_size; 1023 1024 for (i = 0; i < num ; i++) { 1025 rxm = rcv_pkts[i]; 1026 1027 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1028 1029 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1030 PMD_RX_LOG(ERR, "Packet drop"); 1031 nb_enqueued++; 1032 virtio_discard_rxbuf(vq, rxm); 1033 rxvq->stats.errors++; 1034 continue; 1035 } 1036 1037 rxm->port = rxvq->port_id; 1038 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1039 rxm->ol_flags = 0; 1040 rxm->vlan_tci = 0; 1041 1042 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1043 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1044 1045 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1046 RTE_PKTMBUF_HEADROOM - hdr_size); 1047 1048 if (hw->vlan_strip) 1049 rte_vlan_strip(rxm); 1050 1051 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1052 virtio_discard_rxbuf(vq, rxm); 1053 rxvq->stats.errors++; 1054 continue; 1055 } 1056 1057 virtio_rx_stats_updated(rxvq, rxm); 1058 1059 rx_pkts[nb_rx++] = rxm; 1060 } 1061 1062 rxvq->stats.packets += nb_rx; 1063 1064 /* Allocate new mbuf for the used descriptor */ 1065 if (likely(!virtqueue_full(vq))) { 1066 uint16_t free_cnt = vq->vq_free_cnt; 1067 struct rte_mbuf *new_pkts[free_cnt]; 1068 1069 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1070 free_cnt) == 0)) { 1071 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1072 free_cnt); 1073 if (unlikely(error)) { 1074 for (i = 0; i < free_cnt; i++) 1075 rte_pktmbuf_free(new_pkts[i]); 1076 } 1077 nb_enqueued += free_cnt; 1078 } else { 1079 struct rte_eth_dev *dev = 1080 &rte_eth_devices[rxvq->port_id]; 1081 dev->data->rx_mbuf_alloc_failed += free_cnt; 1082 } 1083 } 1084 1085 if (likely(nb_enqueued)) { 1086 vq_update_avail_idx(vq); 1087 1088 if (unlikely(virtqueue_kick_prepare(vq))) { 1089 virtqueue_notify(vq); 1090 PMD_RX_LOG(DEBUG, "Notified"); 1091 } 1092 } 1093 1094 return nb_rx; 1095 } 1096 1097 uint16_t 1098 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1099 uint16_t nb_pkts) 1100 { 1101 struct virtnet_rx *rxvq = rx_queue; 1102 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1103 struct virtio_hw *hw = vq->hw; 1104 struct rte_mbuf *rxm; 1105 uint16_t num, nb_rx; 1106 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1107 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1108 int error; 1109 uint32_t i, nb_enqueued; 1110 uint32_t hdr_size; 1111 struct virtio_net_hdr *hdr; 1112 1113 nb_rx = 0; 1114 if (unlikely(hw->started == 0)) 1115 return nb_rx; 1116 1117 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1118 if (likely(num > DESC_PER_CACHELINE)) 1119 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1120 1121 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1122 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1123 1124 nb_enqueued = 0; 1125 hdr_size = hw->vtnet_hdr_size; 1126 1127 for (i = 0; i < num; i++) { 1128 rxm = rcv_pkts[i]; 1129 1130 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1131 1132 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1133 PMD_RX_LOG(ERR, "Packet drop"); 1134 nb_enqueued++; 1135 virtio_discard_rxbuf(vq, rxm); 1136 rxvq->stats.errors++; 1137 continue; 1138 } 1139 1140 rxm->port = rxvq->port_id; 1141 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1142 rxm->ol_flags = 0; 1143 rxm->vlan_tci = 0; 1144 1145 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1146 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1147 1148 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1149 RTE_PKTMBUF_HEADROOM - hdr_size); 1150 1151 if (hw->vlan_strip) 1152 rte_vlan_strip(rxm); 1153 1154 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1155 virtio_discard_rxbuf(vq, rxm); 1156 rxvq->stats.errors++; 1157 continue; 1158 } 1159 1160 virtio_rx_stats_updated(rxvq, rxm); 1161 1162 rx_pkts[nb_rx++] = rxm; 1163 } 1164 1165 rxvq->stats.packets += nb_rx; 1166 1167 /* Allocate new mbuf for the used descriptor */ 1168 if (likely(!virtqueue_full(vq))) { 1169 uint16_t free_cnt = vq->vq_free_cnt; 1170 struct rte_mbuf *new_pkts[free_cnt]; 1171 1172 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1173 free_cnt) == 0)) { 1174 error = virtqueue_enqueue_recv_refill_packed(vq, 1175 new_pkts, free_cnt); 1176 if (unlikely(error)) { 1177 for (i = 0; i < free_cnt; i++) 1178 rte_pktmbuf_free(new_pkts[i]); 1179 } 1180 nb_enqueued += free_cnt; 1181 } else { 1182 struct rte_eth_dev *dev = 1183 &rte_eth_devices[rxvq->port_id]; 1184 dev->data->rx_mbuf_alloc_failed += free_cnt; 1185 } 1186 } 1187 1188 if (likely(nb_enqueued)) { 1189 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1190 virtqueue_notify(vq); 1191 PMD_RX_LOG(DEBUG, "Notified"); 1192 } 1193 } 1194 1195 return nb_rx; 1196 } 1197 1198 1199 uint16_t 1200 virtio_recv_pkts_inorder(void *rx_queue, 1201 struct rte_mbuf **rx_pkts, 1202 uint16_t nb_pkts) 1203 { 1204 struct virtnet_rx *rxvq = rx_queue; 1205 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1206 struct virtio_hw *hw = vq->hw; 1207 struct rte_mbuf *rxm; 1208 struct rte_mbuf *prev = NULL; 1209 uint16_t nb_used, num, nb_rx; 1210 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1211 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1212 int error; 1213 uint32_t nb_enqueued; 1214 uint32_t seg_num; 1215 uint32_t seg_res; 1216 uint32_t hdr_size; 1217 int32_t i; 1218 1219 nb_rx = 0; 1220 if (unlikely(hw->started == 0)) 1221 return nb_rx; 1222 1223 nb_used = virtqueue_nused(vq); 1224 nb_used = RTE_MIN(nb_used, nb_pkts); 1225 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1226 1227 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1228 1229 nb_enqueued = 0; 1230 seg_num = 1; 1231 seg_res = 0; 1232 hdr_size = hw->vtnet_hdr_size; 1233 1234 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1235 1236 for (i = 0; i < num; i++) { 1237 struct virtio_net_hdr_mrg_rxbuf *header; 1238 1239 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1240 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1241 1242 rxm = rcv_pkts[i]; 1243 1244 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1245 PMD_RX_LOG(ERR, "Packet drop"); 1246 nb_enqueued++; 1247 virtio_discard_rxbuf_inorder(vq, rxm); 1248 rxvq->stats.errors++; 1249 continue; 1250 } 1251 1252 header = (struct virtio_net_hdr_mrg_rxbuf *) 1253 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1254 - hdr_size); 1255 1256 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1257 seg_num = header->num_buffers; 1258 if (seg_num == 0) 1259 seg_num = 1; 1260 } else { 1261 seg_num = 1; 1262 } 1263 1264 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1265 rxm->nb_segs = seg_num; 1266 rxm->ol_flags = 0; 1267 rxm->vlan_tci = 0; 1268 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1269 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1270 1271 rxm->port = rxvq->port_id; 1272 1273 rx_pkts[nb_rx] = rxm; 1274 prev = rxm; 1275 1276 if (vq->hw->has_rx_offload && 1277 virtio_rx_offload(rxm, &header->hdr) < 0) { 1278 virtio_discard_rxbuf_inorder(vq, rxm); 1279 rxvq->stats.errors++; 1280 continue; 1281 } 1282 1283 if (hw->vlan_strip) 1284 rte_vlan_strip(rx_pkts[nb_rx]); 1285 1286 seg_res = seg_num - 1; 1287 1288 /* Merge remaining segments */ 1289 while (seg_res != 0 && i < (num - 1)) { 1290 i++; 1291 1292 rxm = rcv_pkts[i]; 1293 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1294 rxm->pkt_len = (uint32_t)(len[i]); 1295 rxm->data_len = (uint16_t)(len[i]); 1296 1297 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1298 1299 prev->next = rxm; 1300 prev = rxm; 1301 seg_res -= 1; 1302 } 1303 1304 if (!seg_res) { 1305 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1306 nb_rx++; 1307 } 1308 } 1309 1310 /* Last packet still need merge segments */ 1311 while (seg_res != 0) { 1312 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1313 VIRTIO_MBUF_BURST_SZ); 1314 1315 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1316 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1317 rcv_cnt); 1318 uint16_t extra_idx = 0; 1319 1320 rcv_cnt = num; 1321 while (extra_idx < rcv_cnt) { 1322 rxm = rcv_pkts[extra_idx]; 1323 rxm->data_off = 1324 RTE_PKTMBUF_HEADROOM - hdr_size; 1325 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1326 rxm->data_len = (uint16_t)(len[extra_idx]); 1327 prev->next = rxm; 1328 prev = rxm; 1329 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1330 extra_idx += 1; 1331 }; 1332 seg_res -= rcv_cnt; 1333 1334 if (!seg_res) { 1335 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1336 nb_rx++; 1337 } 1338 } else { 1339 PMD_RX_LOG(ERR, 1340 "No enough segments for packet."); 1341 rte_pktmbuf_free(rx_pkts[nb_rx]); 1342 rxvq->stats.errors++; 1343 break; 1344 } 1345 } 1346 1347 rxvq->stats.packets += nb_rx; 1348 1349 /* Allocate new mbuf for the used descriptor */ 1350 1351 if (likely(!virtqueue_full(vq))) { 1352 /* free_cnt may include mrg descs */ 1353 uint16_t free_cnt = vq->vq_free_cnt; 1354 struct rte_mbuf *new_pkts[free_cnt]; 1355 1356 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1357 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1358 free_cnt); 1359 if (unlikely(error)) { 1360 for (i = 0; i < free_cnt; i++) 1361 rte_pktmbuf_free(new_pkts[i]); 1362 } 1363 nb_enqueued += free_cnt; 1364 } else { 1365 struct rte_eth_dev *dev = 1366 &rte_eth_devices[rxvq->port_id]; 1367 dev->data->rx_mbuf_alloc_failed += free_cnt; 1368 } 1369 } 1370 1371 if (likely(nb_enqueued)) { 1372 vq_update_avail_idx(vq); 1373 1374 if (unlikely(virtqueue_kick_prepare(vq))) { 1375 virtqueue_notify(vq); 1376 PMD_RX_LOG(DEBUG, "Notified"); 1377 } 1378 } 1379 1380 return nb_rx; 1381 } 1382 1383 uint16_t 1384 virtio_recv_mergeable_pkts(void *rx_queue, 1385 struct rte_mbuf **rx_pkts, 1386 uint16_t nb_pkts) 1387 { 1388 struct virtnet_rx *rxvq = rx_queue; 1389 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1390 struct virtio_hw *hw = vq->hw; 1391 struct rte_mbuf *rxm; 1392 struct rte_mbuf *prev = NULL; 1393 uint16_t nb_used, num, nb_rx = 0; 1394 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1395 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1396 int error; 1397 uint32_t nb_enqueued = 0; 1398 uint32_t seg_num = 0; 1399 uint32_t seg_res = 0; 1400 uint32_t hdr_size = hw->vtnet_hdr_size; 1401 int32_t i; 1402 1403 if (unlikely(hw->started == 0)) 1404 return nb_rx; 1405 1406 nb_used = virtqueue_nused(vq); 1407 1408 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1409 1410 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1411 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1412 num = VIRTIO_MBUF_BURST_SZ; 1413 if (likely(num > DESC_PER_CACHELINE)) 1414 num = num - ((vq->vq_used_cons_idx + num) % 1415 DESC_PER_CACHELINE); 1416 1417 1418 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1419 1420 for (i = 0; i < num; i++) { 1421 struct virtio_net_hdr_mrg_rxbuf *header; 1422 1423 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1424 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1425 1426 rxm = rcv_pkts[i]; 1427 1428 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1429 PMD_RX_LOG(ERR, "Packet drop"); 1430 nb_enqueued++; 1431 virtio_discard_rxbuf(vq, rxm); 1432 rxvq->stats.errors++; 1433 continue; 1434 } 1435 1436 header = (struct virtio_net_hdr_mrg_rxbuf *) 1437 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1438 - hdr_size); 1439 seg_num = header->num_buffers; 1440 if (seg_num == 0) 1441 seg_num = 1; 1442 1443 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1444 rxm->nb_segs = seg_num; 1445 rxm->ol_flags = 0; 1446 rxm->vlan_tci = 0; 1447 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1448 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1449 1450 rxm->port = rxvq->port_id; 1451 1452 rx_pkts[nb_rx] = rxm; 1453 prev = rxm; 1454 1455 if (hw->has_rx_offload && 1456 virtio_rx_offload(rxm, &header->hdr) < 0) { 1457 virtio_discard_rxbuf(vq, rxm); 1458 rxvq->stats.errors++; 1459 continue; 1460 } 1461 1462 if (hw->vlan_strip) 1463 rte_vlan_strip(rx_pkts[nb_rx]); 1464 1465 seg_res = seg_num - 1; 1466 1467 /* Merge remaining segments */ 1468 while (seg_res != 0 && i < (num - 1)) { 1469 i++; 1470 1471 rxm = rcv_pkts[i]; 1472 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1473 rxm->pkt_len = (uint32_t)(len[i]); 1474 rxm->data_len = (uint16_t)(len[i]); 1475 1476 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1477 1478 prev->next = rxm; 1479 prev = rxm; 1480 seg_res -= 1; 1481 } 1482 1483 if (!seg_res) { 1484 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1485 nb_rx++; 1486 } 1487 } 1488 1489 /* Last packet still need merge segments */ 1490 while (seg_res != 0) { 1491 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1492 VIRTIO_MBUF_BURST_SZ); 1493 1494 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1495 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1496 rcv_cnt); 1497 uint16_t extra_idx = 0; 1498 1499 rcv_cnt = num; 1500 while (extra_idx < rcv_cnt) { 1501 rxm = rcv_pkts[extra_idx]; 1502 rxm->data_off = 1503 RTE_PKTMBUF_HEADROOM - hdr_size; 1504 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1505 rxm->data_len = (uint16_t)(len[extra_idx]); 1506 prev->next = rxm; 1507 prev = rxm; 1508 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1509 extra_idx += 1; 1510 }; 1511 seg_res -= rcv_cnt; 1512 1513 if (!seg_res) { 1514 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1515 nb_rx++; 1516 } 1517 } else { 1518 PMD_RX_LOG(ERR, 1519 "No enough segments for packet."); 1520 rte_pktmbuf_free(rx_pkts[nb_rx]); 1521 rxvq->stats.errors++; 1522 break; 1523 } 1524 } 1525 1526 rxvq->stats.packets += nb_rx; 1527 1528 /* Allocate new mbuf for the used descriptor */ 1529 if (likely(!virtqueue_full(vq))) { 1530 /* free_cnt may include mrg descs */ 1531 uint16_t free_cnt = vq->vq_free_cnt; 1532 struct rte_mbuf *new_pkts[free_cnt]; 1533 1534 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1535 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1536 free_cnt); 1537 if (unlikely(error)) { 1538 for (i = 0; i < free_cnt; i++) 1539 rte_pktmbuf_free(new_pkts[i]); 1540 } 1541 nb_enqueued += free_cnt; 1542 } else { 1543 struct rte_eth_dev *dev = 1544 &rte_eth_devices[rxvq->port_id]; 1545 dev->data->rx_mbuf_alloc_failed += free_cnt; 1546 } 1547 } 1548 1549 if (likely(nb_enqueued)) { 1550 vq_update_avail_idx(vq); 1551 1552 if (unlikely(virtqueue_kick_prepare(vq))) { 1553 virtqueue_notify(vq); 1554 PMD_RX_LOG(DEBUG, "Notified"); 1555 } 1556 } 1557 1558 return nb_rx; 1559 } 1560 1561 uint16_t 1562 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1563 struct rte_mbuf **rx_pkts, 1564 uint16_t nb_pkts) 1565 { 1566 struct virtnet_rx *rxvq = rx_queue; 1567 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1568 struct virtio_hw *hw = vq->hw; 1569 struct rte_mbuf *rxm; 1570 struct rte_mbuf *prev = NULL; 1571 uint16_t num, nb_rx = 0; 1572 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1573 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1574 uint32_t nb_enqueued = 0; 1575 uint32_t seg_num = 0; 1576 uint32_t seg_res = 0; 1577 uint32_t hdr_size = hw->vtnet_hdr_size; 1578 int32_t i; 1579 int error; 1580 1581 if (unlikely(hw->started == 0)) 1582 return nb_rx; 1583 1584 1585 num = nb_pkts; 1586 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1587 num = VIRTIO_MBUF_BURST_SZ; 1588 if (likely(num > DESC_PER_CACHELINE)) 1589 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1590 1591 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1592 1593 for (i = 0; i < num; i++) { 1594 struct virtio_net_hdr_mrg_rxbuf *header; 1595 1596 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1597 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1598 1599 rxm = rcv_pkts[i]; 1600 1601 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1602 PMD_RX_LOG(ERR, "Packet drop"); 1603 nb_enqueued++; 1604 virtio_discard_rxbuf(vq, rxm); 1605 rxvq->stats.errors++; 1606 continue; 1607 } 1608 1609 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1610 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1611 seg_num = header->num_buffers; 1612 1613 if (seg_num == 0) 1614 seg_num = 1; 1615 1616 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1617 rxm->nb_segs = seg_num; 1618 rxm->ol_flags = 0; 1619 rxm->vlan_tci = 0; 1620 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1621 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1622 1623 rxm->port = rxvq->port_id; 1624 rx_pkts[nb_rx] = rxm; 1625 prev = rxm; 1626 1627 if (hw->has_rx_offload && 1628 virtio_rx_offload(rxm, &header->hdr) < 0) { 1629 virtio_discard_rxbuf(vq, rxm); 1630 rxvq->stats.errors++; 1631 continue; 1632 } 1633 1634 if (hw->vlan_strip) 1635 rte_vlan_strip(rx_pkts[nb_rx]); 1636 1637 seg_res = seg_num - 1; 1638 1639 /* Merge remaining segments */ 1640 while (seg_res != 0 && i < (num - 1)) { 1641 i++; 1642 1643 rxm = rcv_pkts[i]; 1644 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1645 rxm->pkt_len = (uint32_t)(len[i]); 1646 rxm->data_len = (uint16_t)(len[i]); 1647 1648 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1649 1650 prev->next = rxm; 1651 prev = rxm; 1652 seg_res -= 1; 1653 } 1654 1655 if (!seg_res) { 1656 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1657 nb_rx++; 1658 } 1659 } 1660 1661 /* Last packet still need merge segments */ 1662 while (seg_res != 0) { 1663 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1664 VIRTIO_MBUF_BURST_SZ); 1665 uint16_t extra_idx = 0; 1666 1667 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1668 len, rcv_cnt); 1669 if (unlikely(rcv_cnt == 0)) { 1670 PMD_RX_LOG(ERR, "No enough segments for packet."); 1671 rte_pktmbuf_free(rx_pkts[nb_rx]); 1672 rxvq->stats.errors++; 1673 break; 1674 } 1675 1676 while (extra_idx < rcv_cnt) { 1677 rxm = rcv_pkts[extra_idx]; 1678 1679 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1680 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1681 rxm->data_len = (uint16_t)(len[extra_idx]); 1682 1683 prev->next = rxm; 1684 prev = rxm; 1685 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1686 extra_idx += 1; 1687 } 1688 seg_res -= rcv_cnt; 1689 if (!seg_res) { 1690 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1691 nb_rx++; 1692 } 1693 } 1694 1695 rxvq->stats.packets += nb_rx; 1696 1697 /* Allocate new mbuf for the used descriptor */ 1698 if (likely(!virtqueue_full(vq))) { 1699 /* free_cnt may include mrg descs */ 1700 uint16_t free_cnt = vq->vq_free_cnt; 1701 struct rte_mbuf *new_pkts[free_cnt]; 1702 1703 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1704 error = virtqueue_enqueue_recv_refill_packed(vq, 1705 new_pkts, free_cnt); 1706 if (unlikely(error)) { 1707 for (i = 0; i < free_cnt; i++) 1708 rte_pktmbuf_free(new_pkts[i]); 1709 } 1710 nb_enqueued += free_cnt; 1711 } else { 1712 struct rte_eth_dev *dev = 1713 &rte_eth_devices[rxvq->port_id]; 1714 dev->data->rx_mbuf_alloc_failed += free_cnt; 1715 } 1716 } 1717 1718 if (likely(nb_enqueued)) { 1719 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1720 virtqueue_notify(vq); 1721 PMD_RX_LOG(DEBUG, "Notified"); 1722 } 1723 } 1724 1725 return nb_rx; 1726 } 1727 1728 uint16_t 1729 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1730 uint16_t nb_pkts) 1731 { 1732 uint16_t nb_tx; 1733 int error; 1734 1735 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1736 struct rte_mbuf *m = tx_pkts[nb_tx]; 1737 1738 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1739 error = rte_validate_tx_offload(m); 1740 if (unlikely(error)) { 1741 rte_errno = -error; 1742 break; 1743 } 1744 #endif 1745 1746 /* Do VLAN tag insertion */ 1747 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) { 1748 error = rte_vlan_insert(&m); 1749 /* rte_vlan_insert() may change pointer 1750 * even in the case of failure 1751 */ 1752 tx_pkts[nb_tx] = m; 1753 1754 if (unlikely(error)) { 1755 rte_errno = -error; 1756 break; 1757 } 1758 } 1759 1760 error = rte_net_intel_cksum_prepare(m); 1761 if (unlikely(error)) { 1762 rte_errno = -error; 1763 break; 1764 } 1765 1766 if (m->ol_flags & PKT_TX_TCP_SEG) 1767 virtio_tso_fix_cksum(m); 1768 } 1769 1770 return nb_tx; 1771 } 1772 1773 uint16_t 1774 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 1775 uint16_t nb_pkts) 1776 { 1777 struct virtnet_tx *txvq = tx_queue; 1778 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1779 struct virtio_hw *hw = vq->hw; 1780 uint16_t hdr_size = hw->vtnet_hdr_size; 1781 uint16_t nb_tx = 0; 1782 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 1783 1784 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1785 return nb_tx; 1786 1787 if (unlikely(nb_pkts < 1)) 1788 return nb_pkts; 1789 1790 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1791 1792 if (nb_pkts > vq->vq_free_cnt) 1793 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 1794 in_order); 1795 1796 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1797 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1798 int can_push = 0, use_indirect = 0, slots, need; 1799 1800 /* optimize ring usage */ 1801 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1802 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1803 rte_mbuf_refcnt_read(txm) == 1 && 1804 RTE_MBUF_DIRECT(txm) && 1805 txm->nb_segs == 1 && 1806 rte_pktmbuf_headroom(txm) >= hdr_size && 1807 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1808 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1809 can_push = 1; 1810 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1811 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1812 use_indirect = 1; 1813 /* How many main ring entries are needed to this Tx? 1814 * indirect => 1 1815 * any_layout => number of segments 1816 * default => number of segments + 1 1817 */ 1818 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1819 need = slots - vq->vq_free_cnt; 1820 1821 /* Positive value indicates it need free vring descriptors */ 1822 if (unlikely(need > 0)) { 1823 virtio_xmit_cleanup_packed(vq, need, in_order); 1824 need = slots - vq->vq_free_cnt; 1825 if (unlikely(need > 0)) { 1826 PMD_TX_LOG(ERR, 1827 "No free tx descriptors to transmit"); 1828 break; 1829 } 1830 } 1831 1832 /* Enqueue Packet buffers */ 1833 if (can_push) 1834 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 1835 else 1836 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 1837 use_indirect, 0, 1838 in_order); 1839 1840 virtio_update_packet_stats(&txvq->stats, txm); 1841 } 1842 1843 txvq->stats.packets += nb_tx; 1844 1845 if (likely(nb_tx)) { 1846 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1847 virtqueue_notify(vq); 1848 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1849 } 1850 } 1851 1852 return nb_tx; 1853 } 1854 1855 uint16_t 1856 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1857 { 1858 struct virtnet_tx *txvq = tx_queue; 1859 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1860 struct virtio_hw *hw = vq->hw; 1861 uint16_t hdr_size = hw->vtnet_hdr_size; 1862 uint16_t nb_used, nb_tx = 0; 1863 1864 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1865 return nb_tx; 1866 1867 if (unlikely(nb_pkts < 1)) 1868 return nb_pkts; 1869 1870 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1871 1872 nb_used = virtqueue_nused(vq); 1873 1874 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1875 virtio_xmit_cleanup(vq, nb_used); 1876 1877 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1878 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1879 int can_push = 0, use_indirect = 0, slots, need; 1880 1881 /* optimize ring usage */ 1882 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1883 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1884 rte_mbuf_refcnt_read(txm) == 1 && 1885 RTE_MBUF_DIRECT(txm) && 1886 txm->nb_segs == 1 && 1887 rte_pktmbuf_headroom(txm) >= hdr_size && 1888 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1889 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1890 can_push = 1; 1891 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1892 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1893 use_indirect = 1; 1894 1895 /* How many main ring entries are needed to this Tx? 1896 * any_layout => number of segments 1897 * indirect => 1 1898 * default => number of segments + 1 1899 */ 1900 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1901 need = slots - vq->vq_free_cnt; 1902 1903 /* Positive value indicates it need free vring descriptors */ 1904 if (unlikely(need > 0)) { 1905 nb_used = virtqueue_nused(vq); 1906 1907 need = RTE_MIN(need, (int)nb_used); 1908 1909 virtio_xmit_cleanup(vq, need); 1910 need = slots - vq->vq_free_cnt; 1911 if (unlikely(need > 0)) { 1912 PMD_TX_LOG(ERR, 1913 "No free tx descriptors to transmit"); 1914 break; 1915 } 1916 } 1917 1918 /* Enqueue Packet buffers */ 1919 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 1920 can_push, 0); 1921 1922 virtio_update_packet_stats(&txvq->stats, txm); 1923 } 1924 1925 txvq->stats.packets += nb_tx; 1926 1927 if (likely(nb_tx)) { 1928 vq_update_avail_idx(vq); 1929 1930 if (unlikely(virtqueue_kick_prepare(vq))) { 1931 virtqueue_notify(vq); 1932 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1933 } 1934 } 1935 1936 return nb_tx; 1937 } 1938 1939 static __rte_always_inline int 1940 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 1941 { 1942 uint16_t nb_used, nb_clean, nb_descs; 1943 1944 nb_descs = vq->vq_free_cnt + need; 1945 nb_used = virtqueue_nused(vq); 1946 nb_clean = RTE_MIN(need, (int)nb_used); 1947 1948 virtio_xmit_cleanup_inorder(vq, nb_clean); 1949 1950 return nb_descs - vq->vq_free_cnt; 1951 } 1952 1953 uint16_t 1954 virtio_xmit_pkts_inorder(void *tx_queue, 1955 struct rte_mbuf **tx_pkts, 1956 uint16_t nb_pkts) 1957 { 1958 struct virtnet_tx *txvq = tx_queue; 1959 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1960 struct virtio_hw *hw = vq->hw; 1961 uint16_t hdr_size = hw->vtnet_hdr_size; 1962 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 1963 struct rte_mbuf *inorder_pkts[nb_pkts]; 1964 int need; 1965 1966 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1967 return nb_tx; 1968 1969 if (unlikely(nb_pkts < 1)) 1970 return nb_pkts; 1971 1972 VIRTQUEUE_DUMP(vq); 1973 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1974 nb_used = virtqueue_nused(vq); 1975 1976 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1977 virtio_xmit_cleanup_inorder(vq, nb_used); 1978 1979 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1980 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1981 int slots; 1982 1983 /* optimize ring usage */ 1984 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1985 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1986 rte_mbuf_refcnt_read(txm) == 1 && 1987 RTE_MBUF_DIRECT(txm) && 1988 txm->nb_segs == 1 && 1989 rte_pktmbuf_headroom(txm) >= hdr_size && 1990 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1991 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) { 1992 inorder_pkts[nb_inorder_pkts] = txm; 1993 nb_inorder_pkts++; 1994 1995 continue; 1996 } 1997 1998 if (nb_inorder_pkts) { 1999 need = nb_inorder_pkts - vq->vq_free_cnt; 2000 if (unlikely(need > 0)) { 2001 need = virtio_xmit_try_cleanup_inorder(vq, 2002 need); 2003 if (unlikely(need > 0)) { 2004 PMD_TX_LOG(ERR, 2005 "No free tx descriptors to " 2006 "transmit"); 2007 break; 2008 } 2009 } 2010 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2011 nb_inorder_pkts); 2012 nb_inorder_pkts = 0; 2013 } 2014 2015 slots = txm->nb_segs + 1; 2016 need = slots - vq->vq_free_cnt; 2017 if (unlikely(need > 0)) { 2018 need = virtio_xmit_try_cleanup_inorder(vq, slots); 2019 2020 if (unlikely(need > 0)) { 2021 PMD_TX_LOG(ERR, 2022 "No free tx descriptors to transmit"); 2023 break; 2024 } 2025 } 2026 /* Enqueue Packet buffers */ 2027 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 2028 2029 virtio_update_packet_stats(&txvq->stats, txm); 2030 } 2031 2032 /* Transmit all inorder packets */ 2033 if (nb_inorder_pkts) { 2034 need = nb_inorder_pkts - vq->vq_free_cnt; 2035 if (unlikely(need > 0)) { 2036 need = virtio_xmit_try_cleanup_inorder(vq, 2037 need); 2038 if (unlikely(need > 0)) { 2039 PMD_TX_LOG(ERR, 2040 "No free tx descriptors to transmit"); 2041 nb_inorder_pkts = vq->vq_free_cnt; 2042 nb_tx -= need; 2043 } 2044 } 2045 2046 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2047 nb_inorder_pkts); 2048 } 2049 2050 txvq->stats.packets += nb_tx; 2051 2052 if (likely(nb_tx)) { 2053 vq_update_avail_idx(vq); 2054 2055 if (unlikely(virtqueue_kick_prepare(vq))) { 2056 virtqueue_notify(vq); 2057 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2058 } 2059 } 2060 2061 VIRTQUEUE_DUMP(vq); 2062 2063 return nb_tx; 2064 } 2065 2066 __rte_weak uint16_t 2067 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, 2068 struct rte_mbuf **rx_pkts __rte_unused, 2069 uint16_t nb_pkts __rte_unused) 2070 { 2071 return 0; 2072 } 2073 2074 __rte_weak uint16_t 2075 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, 2076 struct rte_mbuf **tx_pkts __rte_unused, 2077 uint16_t nb_pkts __rte_unused) 2078 { 2079 return 0; 2080 } 2081