1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_cycles.h> 12 #include <rte_memory.h> 13 #include <rte_branch_prediction.h> 14 #include <rte_mempool.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_ether.h> 18 #include <ethdev_driver.h> 19 #include <rte_prefetch.h> 20 #include <rte_string_fns.h> 21 #include <rte_errno.h> 22 #include <rte_byteorder.h> 23 #include <rte_net.h> 24 #include <rte_ip.h> 25 #include <rte_udp.h> 26 #include <rte_tcp.h> 27 28 #include "virtio_logs.h" 29 #include "virtio_ethdev.h" 30 #include "virtio.h" 31 #include "virtqueue.h" 32 #include "virtio_rxtx.h" 33 #include "virtio_rxtx_simple.h" 34 #include "virtio_ring.h" 35 36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 38 #else 39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 40 #endif 41 42 void 43 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 44 { 45 vq->vq_free_cnt += num; 46 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 47 } 48 49 void 50 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 51 { 52 struct vring_desc *dp, *dp_tail; 53 struct vq_desc_extra *dxp; 54 uint16_t desc_idx_last = desc_idx; 55 56 dp = &vq->vq_split.ring.desc[desc_idx]; 57 dxp = &vq->vq_descx[desc_idx]; 58 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 59 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 60 while (dp->flags & VRING_DESC_F_NEXT) { 61 desc_idx_last = dp->next; 62 dp = &vq->vq_split.ring.desc[dp->next]; 63 } 64 } 65 dxp->ndescs = 0; 66 67 /* 68 * We must append the existing free chain, if any, to the end of 69 * newly freed chain. If the virtqueue was completely used, then 70 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 71 */ 72 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 73 vq->vq_desc_head_idx = desc_idx; 74 } else { 75 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 76 dp_tail->next = desc_idx; 77 } 78 79 vq->vq_desc_tail_idx = desc_idx_last; 80 dp->next = VQ_RING_DESC_CHAIN_END; 81 } 82 83 void 84 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) 85 { 86 uint32_t s = mbuf->pkt_len; 87 struct rte_ether_addr *ea; 88 89 stats->bytes += s; 90 91 if (s == 64) { 92 stats->size_bins[1]++; 93 } else if (s > 64 && s < 1024) { 94 uint32_t bin; 95 96 /* count zeros, and offset into correct bin */ 97 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 98 stats->size_bins[bin]++; 99 } else { 100 if (s < 64) 101 stats->size_bins[0]++; 102 else if (s < 1519) 103 stats->size_bins[6]++; 104 else 105 stats->size_bins[7]++; 106 } 107 108 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 109 if (rte_is_multicast_ether_addr(ea)) { 110 if (rte_is_broadcast_ether_addr(ea)) 111 stats->broadcast++; 112 else 113 stats->multicast++; 114 } 115 } 116 117 static inline void 118 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 119 { 120 VIRTIO_DUMP_PACKET(m, m->data_len); 121 122 virtio_update_packet_stats(&rxvq->stats, m); 123 } 124 125 static uint16_t 126 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 127 struct rte_mbuf **rx_pkts, 128 uint32_t *len, 129 uint16_t num) 130 { 131 struct rte_mbuf *cookie; 132 uint16_t used_idx; 133 uint16_t id; 134 struct vring_packed_desc *desc; 135 uint16_t i; 136 137 desc = vq->vq_packed.ring.desc; 138 139 for (i = 0; i < num; i++) { 140 used_idx = vq->vq_used_cons_idx; 141 /* desc_is_used has a load-acquire or rte_io_rmb inside 142 * and wait for used desc in virtqueue. 143 */ 144 if (!desc_is_used(&desc[used_idx], vq)) 145 return i; 146 len[i] = desc[used_idx].len; 147 id = desc[used_idx].id; 148 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 149 if (unlikely(cookie == NULL)) { 150 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 151 vq->vq_used_cons_idx); 152 break; 153 } 154 rte_prefetch0(cookie); 155 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 156 rx_pkts[i] = cookie; 157 158 vq->vq_free_cnt++; 159 vq->vq_used_cons_idx++; 160 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 161 vq->vq_used_cons_idx -= vq->vq_nentries; 162 vq->vq_packed.used_wrap_counter ^= 1; 163 } 164 } 165 166 return i; 167 } 168 169 static uint16_t 170 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 171 uint32_t *len, uint16_t num) 172 { 173 struct vring_used_elem *uep; 174 struct rte_mbuf *cookie; 175 uint16_t used_idx, desc_idx; 176 uint16_t i; 177 178 /* Caller does the check */ 179 for (i = 0; i < num ; i++) { 180 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 181 uep = &vq->vq_split.ring.used->ring[used_idx]; 182 desc_idx = (uint16_t) uep->id; 183 len[i] = uep->len; 184 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 185 186 if (unlikely(cookie == NULL)) { 187 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 188 vq->vq_used_cons_idx); 189 break; 190 } 191 192 rte_prefetch0(cookie); 193 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 194 rx_pkts[i] = cookie; 195 vq->vq_used_cons_idx++; 196 vq_ring_free_chain(vq, desc_idx); 197 vq->vq_descx[desc_idx].cookie = NULL; 198 } 199 200 return i; 201 } 202 203 static uint16_t 204 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 205 struct rte_mbuf **rx_pkts, 206 uint32_t *len, 207 uint16_t num) 208 { 209 struct vring_used_elem *uep; 210 struct rte_mbuf *cookie; 211 uint16_t used_idx = 0; 212 uint16_t i; 213 214 if (unlikely(num == 0)) 215 return 0; 216 217 for (i = 0; i < num; i++) { 218 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 219 /* Desc idx same as used idx */ 220 uep = &vq->vq_split.ring.used->ring[used_idx]; 221 len[i] = uep->len; 222 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 223 224 if (unlikely(cookie == NULL)) { 225 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 226 vq->vq_used_cons_idx); 227 break; 228 } 229 230 rte_prefetch0(cookie); 231 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 232 rx_pkts[i] = cookie; 233 vq->vq_used_cons_idx++; 234 vq->vq_descx[used_idx].cookie = NULL; 235 } 236 237 vq_ring_free_inorder(vq, used_idx, i); 238 return i; 239 } 240 241 static inline int 242 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 243 struct rte_mbuf **cookies, 244 uint16_t num) 245 { 246 struct vq_desc_extra *dxp; 247 struct virtio_hw *hw = vq->hw; 248 struct vring_desc *start_dp; 249 uint16_t head_idx, idx, i = 0; 250 251 if (unlikely(vq->vq_free_cnt == 0)) 252 return -ENOSPC; 253 if (unlikely(vq->vq_free_cnt < num)) 254 return -EMSGSIZE; 255 256 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 257 start_dp = vq->vq_split.ring.desc; 258 259 while (i < num) { 260 idx = head_idx & (vq->vq_nentries - 1); 261 dxp = &vq->vq_descx[idx]; 262 dxp->cookie = (void *)cookies[i]; 263 dxp->ndescs = 1; 264 265 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) + 266 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 267 start_dp[idx].len = cookies[i]->buf_len - 268 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 269 start_dp[idx].flags = VRING_DESC_F_WRITE; 270 271 vq_update_avail_ring(vq, idx); 272 head_idx++; 273 i++; 274 } 275 276 vq->vq_desc_head_idx += num; 277 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 278 return 0; 279 } 280 281 static inline int 282 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 283 uint16_t num) 284 { 285 struct vq_desc_extra *dxp; 286 struct virtio_hw *hw = vq->hw; 287 struct vring_desc *start_dp = vq->vq_split.ring.desc; 288 uint16_t idx, i; 289 290 if (unlikely(vq->vq_free_cnt == 0)) 291 return -ENOSPC; 292 if (unlikely(vq->vq_free_cnt < num)) 293 return -EMSGSIZE; 294 295 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 296 return -EFAULT; 297 298 for (i = 0; i < num; i++) { 299 idx = vq->vq_desc_head_idx; 300 dxp = &vq->vq_descx[idx]; 301 dxp->cookie = (void *)cookie[i]; 302 dxp->ndescs = 1; 303 304 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + 305 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 306 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + 307 hw->vtnet_hdr_size; 308 start_dp[idx].flags = VRING_DESC_F_WRITE; 309 vq->vq_desc_head_idx = start_dp[idx].next; 310 vq_update_avail_ring(vq, idx); 311 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 312 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 313 break; 314 } 315 } 316 317 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 318 319 return 0; 320 } 321 322 static inline void 323 virtqueue_refill_single_packed(struct virtqueue *vq, 324 struct vring_packed_desc *dp, 325 struct rte_mbuf *cookie) 326 { 327 uint16_t flags = vq->vq_packed.cached_flags; 328 struct virtio_hw *hw = vq->hw; 329 330 dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 331 dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 332 333 virtqueue_store_flags_packed(dp, flags, hw->weak_barriers); 334 335 if (++vq->vq_avail_idx >= vq->vq_nentries) { 336 vq->vq_avail_idx -= vq->vq_nentries; 337 vq->vq_packed.cached_flags ^= 338 VRING_PACKED_DESC_F_AVAIL_USED; 339 flags = vq->vq_packed.cached_flags; 340 } 341 } 342 343 static inline int 344 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq, 345 struct rte_mbuf **cookie, uint16_t num) 346 { 347 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 348 struct vq_desc_extra *dxp; 349 uint16_t idx; 350 int i; 351 352 if (unlikely(vq->vq_free_cnt == 0)) 353 return -ENOSPC; 354 if (unlikely(vq->vq_free_cnt < num)) 355 return -EMSGSIZE; 356 357 for (i = 0; i < num; i++) { 358 idx = vq->vq_avail_idx; 359 dxp = &vq->vq_descx[idx]; 360 dxp->cookie = (void *)cookie[i]; 361 dxp->ndescs = 1; 362 363 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 364 } 365 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 366 return 0; 367 } 368 369 static inline int 370 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 371 struct rte_mbuf **cookie, uint16_t num) 372 { 373 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 374 struct vq_desc_extra *dxp; 375 uint16_t idx, did; 376 int i; 377 378 if (unlikely(vq->vq_free_cnt == 0)) 379 return -ENOSPC; 380 if (unlikely(vq->vq_free_cnt < num)) 381 return -EMSGSIZE; 382 383 for (i = 0; i < num; i++) { 384 idx = vq->vq_avail_idx; 385 did = start_dp[idx].id; 386 dxp = &vq->vq_descx[did]; 387 dxp->cookie = (void *)cookie[i]; 388 dxp->ndescs = 1; 389 390 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 391 } 392 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 393 return 0; 394 } 395 396 /* When doing TSO, the IP length is not included in the pseudo header 397 * checksum of the packet given to the PMD, but for virtio it is 398 * expected. 399 */ 400 static void 401 virtio_tso_fix_cksum(struct rte_mbuf *m) 402 { 403 /* common case: header is not fragmented */ 404 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 405 m->l4_len)) { 406 struct rte_ipv4_hdr *iph; 407 struct rte_ipv6_hdr *ip6h; 408 struct rte_tcp_hdr *th; 409 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; 410 uint32_t tmp; 411 412 iph = rte_pktmbuf_mtod_offset(m, 413 struct rte_ipv4_hdr *, m->l2_len); 414 th = RTE_PTR_ADD(iph, m->l3_len); 415 if ((iph->version_ihl >> 4) == 4) { 416 iph->hdr_checksum = 0; 417 iph->hdr_checksum = rte_ipv4_cksum(iph); 418 ip_len = iph->total_length; 419 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - 420 m->l3_len); 421 } else { 422 ip6h = (struct rte_ipv6_hdr *)iph; 423 ip_paylen = ip6h->payload_len; 424 } 425 426 /* calculate the new phdr checksum not including ip_paylen */ 427 prev_cksum = th->cksum; 428 tmp = prev_cksum; 429 tmp += ip_paylen; 430 tmp = (tmp & 0xffff) + (tmp >> 16); 431 new_cksum = tmp; 432 433 /* replace it in the packet */ 434 th->cksum = new_cksum; 435 } 436 } 437 438 439 440 441 static inline void 442 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 443 struct rte_mbuf **cookies, 444 uint16_t num) 445 { 446 struct vq_desc_extra *dxp; 447 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 448 struct vring_desc *start_dp; 449 struct virtio_net_hdr *hdr; 450 uint16_t idx; 451 int16_t head_size = vq->hw->vtnet_hdr_size; 452 uint16_t i = 0; 453 454 idx = vq->vq_desc_head_idx; 455 start_dp = vq->vq_split.ring.desc; 456 457 while (i < num) { 458 idx = idx & (vq->vq_nentries - 1); 459 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 460 dxp->cookie = (void *)cookies[i]; 461 dxp->ndescs = 1; 462 virtio_update_packet_stats(&txvq->stats, cookies[i]); 463 464 hdr = rte_pktmbuf_mtod_offset(cookies[i], 465 struct virtio_net_hdr *, -head_size); 466 467 /* if offload disabled, hdr is not zeroed yet, do it now */ 468 if (!vq->hw->has_tx_offload) 469 virtqueue_clear_net_hdr(hdr); 470 else 471 virtqueue_xmit_offload(hdr, cookies[i]); 472 473 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size; 474 start_dp[idx].len = cookies[i]->data_len + head_size; 475 start_dp[idx].flags = 0; 476 477 478 vq_update_avail_ring(vq, idx); 479 480 idx++; 481 i++; 482 }; 483 484 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 485 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 486 } 487 488 static inline void 489 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 490 struct rte_mbuf *cookie, 491 int in_order) 492 { 493 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 494 struct vring_packed_desc *dp; 495 struct vq_desc_extra *dxp; 496 uint16_t idx, id, flags; 497 int16_t head_size = vq->hw->vtnet_hdr_size; 498 struct virtio_net_hdr *hdr; 499 500 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 501 idx = vq->vq_avail_idx; 502 dp = &vq->vq_packed.ring.desc[idx]; 503 504 dxp = &vq->vq_descx[id]; 505 dxp->ndescs = 1; 506 dxp->cookie = cookie; 507 508 flags = vq->vq_packed.cached_flags; 509 510 /* prepend cannot fail, checked by caller */ 511 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 512 -head_size); 513 514 /* if offload disabled, hdr is not zeroed yet, do it now */ 515 if (!vq->hw->has_tx_offload) 516 virtqueue_clear_net_hdr(hdr); 517 else 518 virtqueue_xmit_offload(hdr, cookie); 519 520 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; 521 dp->len = cookie->data_len + head_size; 522 dp->id = id; 523 524 if (++vq->vq_avail_idx >= vq->vq_nentries) { 525 vq->vq_avail_idx -= vq->vq_nentries; 526 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 527 } 528 529 vq->vq_free_cnt--; 530 531 if (!in_order) { 532 vq->vq_desc_head_idx = dxp->next; 533 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 534 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 535 } 536 537 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 538 } 539 540 static inline void 541 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 542 uint16_t needed, int use_indirect, int can_push, 543 int in_order) 544 { 545 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 546 struct vq_desc_extra *dxp; 547 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 548 struct vring_desc *start_dp; 549 uint16_t seg_num = cookie->nb_segs; 550 uint16_t head_idx, idx; 551 int16_t head_size = vq->hw->vtnet_hdr_size; 552 bool prepend_header = false; 553 struct virtio_net_hdr *hdr; 554 555 head_idx = vq->vq_desc_head_idx; 556 idx = head_idx; 557 if (in_order) 558 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 559 else 560 dxp = &vq->vq_descx[idx]; 561 dxp->cookie = (void *)cookie; 562 dxp->ndescs = needed; 563 564 start_dp = vq->vq_split.ring.desc; 565 566 if (can_push) { 567 /* prepend cannot fail, checked by caller */ 568 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 569 -head_size); 570 prepend_header = true; 571 572 /* if offload disabled, it is not zeroed below, do it now */ 573 if (!vq->hw->has_tx_offload) 574 virtqueue_clear_net_hdr(hdr); 575 } else if (use_indirect) { 576 /* setup tx ring slot to point to indirect 577 * descriptor list stored in reserved region. 578 * 579 * the first slot in indirect ring is already preset 580 * to point to the header in reserved region 581 */ 582 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 583 RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 584 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 585 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 586 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 587 588 /* loop below will fill in rest of the indirect elements */ 589 start_dp = txr[idx].tx_indir; 590 idx = 1; 591 } else { 592 /* setup first tx ring slot to point to header 593 * stored in reserved region. 594 */ 595 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 596 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 597 start_dp[idx].len = vq->hw->vtnet_hdr_size; 598 start_dp[idx].flags = VRING_DESC_F_NEXT; 599 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 600 601 idx = start_dp[idx].next; 602 } 603 604 if (vq->hw->has_tx_offload) 605 virtqueue_xmit_offload(hdr, cookie); 606 607 do { 608 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 609 start_dp[idx].len = cookie->data_len; 610 if (prepend_header) { 611 start_dp[idx].addr -= head_size; 612 start_dp[idx].len += head_size; 613 prepend_header = false; 614 } 615 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 616 idx = start_dp[idx].next; 617 } while ((cookie = cookie->next) != NULL); 618 619 if (use_indirect) 620 idx = vq->vq_split.ring.desc[head_idx].next; 621 622 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 623 624 vq->vq_desc_head_idx = idx; 625 vq_update_avail_ring(vq, head_idx); 626 627 if (!in_order) { 628 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 629 vq->vq_desc_tail_idx = idx; 630 } 631 } 632 633 void 634 virtio_dev_cq_start(struct rte_eth_dev *dev) 635 { 636 struct virtio_hw *hw = dev->data->dev_private; 637 638 if (hw->cvq) { 639 rte_spinlock_init(&hw->cvq->lock); 640 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); 641 } 642 } 643 644 int 645 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 646 uint16_t queue_idx, 647 uint16_t nb_desc, 648 unsigned int socket_id __rte_unused, 649 const struct rte_eth_rxconf *rx_conf, 650 struct rte_mempool *mp) 651 { 652 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 653 struct virtio_hw *hw = dev->data->dev_private; 654 struct virtqueue *vq = hw->vqs[vq_idx]; 655 struct virtnet_rx *rxvq; 656 uint16_t rx_free_thresh; 657 uint16_t buf_size; 658 const char *error; 659 660 PMD_INIT_FUNC_TRACE(); 661 662 if (rx_conf->rx_deferred_start) { 663 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 664 return -EINVAL; 665 } 666 667 buf_size = virtio_rx_mem_pool_buf_size(mp); 668 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size, 669 hw->rx_ol_scatter, &error)) { 670 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s", 671 queue_idx, error); 672 return -EINVAL; 673 } 674 675 rx_free_thresh = rx_conf->rx_free_thresh; 676 if (rx_free_thresh == 0) 677 rx_free_thresh = 678 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); 679 680 if (rx_free_thresh & 0x3) { 681 PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four." 682 " (rx_free_thresh=%u port=%u queue=%u)", 683 rx_free_thresh, dev->data->port_id, queue_idx); 684 return -EINVAL; 685 } 686 687 if (rx_free_thresh >= vq->vq_nentries) { 688 PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the " 689 "number of RX entries (%u)." 690 " (rx_free_thresh=%u port=%u queue=%u)", 691 vq->vq_nentries, 692 rx_free_thresh, dev->data->port_id, queue_idx); 693 return -EINVAL; 694 } 695 vq->vq_free_thresh = rx_free_thresh; 696 697 /* 698 * For split ring vectorized path descriptors number must be 699 * equal to the ring size. 700 */ 701 if (nb_desc > vq->vq_nentries || 702 (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) { 703 nb_desc = vq->vq_nentries; 704 } 705 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 706 707 rxvq = &vq->rxq; 708 rxvq->queue_id = queue_idx; 709 rxvq->mpool = mp; 710 dev->data->rx_queues[queue_idx] = rxvq; 711 712 return 0; 713 } 714 715 int 716 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 717 { 718 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 719 struct virtio_hw *hw = dev->data->dev_private; 720 struct virtqueue *vq = hw->vqs[vq_idx]; 721 struct virtnet_rx *rxvq = &vq->rxq; 722 struct rte_mbuf *m; 723 uint16_t desc_idx; 724 int error, nbufs, i; 725 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 726 727 PMD_INIT_FUNC_TRACE(); 728 729 /* Allocate blank mbufs for the each rx descriptor */ 730 nbufs = 0; 731 732 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 733 for (desc_idx = 0; desc_idx < vq->vq_nentries; 734 desc_idx++) { 735 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 736 vq->vq_split.ring.desc[desc_idx].flags = 737 VRING_DESC_F_WRITE; 738 } 739 740 virtio_rxq_vec_setup(rxvq); 741 } 742 743 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); 744 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) 745 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; 746 747 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 748 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 749 virtio_rxq_rearm_vec(rxvq); 750 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 751 } 752 } else if (!virtio_with_packed_queue(vq->hw) && in_order) { 753 if ((!virtqueue_full(vq))) { 754 uint16_t free_cnt = vq->vq_free_cnt; 755 struct rte_mbuf *pkts[free_cnt]; 756 757 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 758 free_cnt)) { 759 error = virtqueue_enqueue_refill_inorder(vq, 760 pkts, 761 free_cnt); 762 if (unlikely(error)) { 763 for (i = 0; i < free_cnt; i++) 764 rte_pktmbuf_free(pkts[i]); 765 } else { 766 nbufs += free_cnt; 767 } 768 } 769 770 vq_update_avail_idx(vq); 771 } 772 } else { 773 while (!virtqueue_full(vq)) { 774 m = rte_mbuf_raw_alloc(rxvq->mpool); 775 if (m == NULL) 776 break; 777 778 /* Enqueue allocated buffers */ 779 if (virtio_with_packed_queue(vq->hw)) 780 error = virtqueue_enqueue_recv_refill_packed_init(vq, 781 &m, 1); 782 else 783 error = virtqueue_enqueue_recv_refill(vq, 784 &m, 1); 785 if (error) { 786 rte_pktmbuf_free(m); 787 break; 788 } 789 nbufs++; 790 } 791 792 if (!virtio_with_packed_queue(vq->hw)) 793 vq_update_avail_idx(vq); 794 } 795 796 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); 797 798 VIRTQUEUE_DUMP(vq); 799 800 return 0; 801 } 802 803 /* 804 * struct rte_eth_dev *dev: Used to update dev 805 * uint16_t nb_desc: Defaults to values read from config space 806 * unsigned int socket_id: Used to allocate memzone 807 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 808 * uint16_t queue_idx: Just used as an index in dev txq list 809 */ 810 int 811 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 812 uint16_t queue_idx, 813 uint16_t nb_desc, 814 unsigned int socket_id __rte_unused, 815 const struct rte_eth_txconf *tx_conf) 816 { 817 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 818 struct virtio_hw *hw = dev->data->dev_private; 819 struct virtqueue *vq = hw->vqs[vq_idx]; 820 struct virtnet_tx *txvq; 821 uint16_t tx_free_thresh; 822 823 PMD_INIT_FUNC_TRACE(); 824 825 if (tx_conf->tx_deferred_start) { 826 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 827 return -EINVAL; 828 } 829 830 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 831 nb_desc = vq->vq_nentries; 832 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 833 834 txvq = &vq->txq; 835 txvq->queue_id = queue_idx; 836 837 tx_free_thresh = tx_conf->tx_free_thresh; 838 if (tx_free_thresh == 0) 839 tx_free_thresh = 840 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 841 842 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 843 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " 844 "number of TX entries minus 3 (%u)." 845 " (tx_free_thresh=%u port=%u queue=%u)", 846 vq->vq_nentries - 3, 847 tx_free_thresh, dev->data->port_id, queue_idx); 848 return -EINVAL; 849 } 850 851 vq->vq_free_thresh = tx_free_thresh; 852 853 dev->data->tx_queues[queue_idx] = txvq; 854 return 0; 855 } 856 857 int 858 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 859 uint16_t queue_idx) 860 { 861 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 862 struct virtio_hw *hw = dev->data->dev_private; 863 struct virtqueue *vq = hw->vqs[vq_idx]; 864 865 PMD_INIT_FUNC_TRACE(); 866 867 if (!virtio_with_packed_queue(hw)) { 868 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 869 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 870 } 871 872 VIRTQUEUE_DUMP(vq); 873 874 return 0; 875 } 876 877 static inline void 878 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 879 { 880 int error; 881 /* 882 * Requeue the discarded mbuf. This should always be 883 * successful since it was just dequeued. 884 */ 885 if (virtio_with_packed_queue(vq->hw)) 886 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 887 else 888 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 889 890 if (unlikely(error)) { 891 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 892 rte_pktmbuf_free(m); 893 } 894 } 895 896 static inline void 897 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 898 { 899 int error; 900 901 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 902 if (unlikely(error)) { 903 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 904 rte_pktmbuf_free(m); 905 } 906 } 907 908 /* Optionally fill offload information in structure */ 909 static inline int 910 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 911 { 912 struct rte_net_hdr_lens hdr_lens; 913 uint32_t hdrlen, ptype; 914 int l4_supported = 0; 915 916 /* nothing to do */ 917 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 918 return 0; 919 920 m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN; 921 922 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 923 m->packet_type = ptype; 924 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 925 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 926 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 927 l4_supported = 1; 928 929 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 930 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 931 if (hdr->csum_start <= hdrlen && l4_supported) { 932 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_NONE; 933 } else { 934 /* Unknown proto or tunnel, do sw cksum. We can assume 935 * the cksum field is in the first segment since the 936 * buffers we provided to the host are large enough. 937 * In case of SCTP, this will be wrong since it's a CRC 938 * but there's nothing we can do. 939 */ 940 uint16_t csum = 0, off; 941 942 if (rte_raw_cksum_mbuf(m, hdr->csum_start, 943 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 944 &csum) < 0) 945 return -EINVAL; 946 if (likely(csum != 0xffff)) 947 csum = ~csum; 948 off = hdr->csum_offset + hdr->csum_start; 949 if (rte_pktmbuf_data_len(m) >= off + 1) 950 *rte_pktmbuf_mtod_offset(m, uint16_t *, 951 off) = csum; 952 } 953 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 954 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 955 } 956 957 /* GSO request, save required information in mbuf */ 958 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 959 /* Check unsupported modes */ 960 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 961 (hdr->gso_size == 0)) { 962 return -EINVAL; 963 } 964 965 /* Update mss lengthes in mbuf */ 966 m->tso_segsz = hdr->gso_size; 967 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 968 case VIRTIO_NET_HDR_GSO_TCPV4: 969 case VIRTIO_NET_HDR_GSO_TCPV6: 970 m->ol_flags |= RTE_MBUF_F_RX_LRO | 971 RTE_MBUF_F_RX_L4_CKSUM_NONE; 972 break; 973 default: 974 return -EINVAL; 975 } 976 } 977 978 return 0; 979 } 980 981 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 982 uint16_t 983 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 984 { 985 struct virtnet_rx *rxvq = rx_queue; 986 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 987 struct virtio_hw *hw = vq->hw; 988 struct rte_mbuf *rxm; 989 uint16_t nb_used, num, nb_rx; 990 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 991 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 992 int error; 993 uint32_t i, nb_enqueued; 994 uint32_t hdr_size; 995 struct virtio_net_hdr *hdr; 996 997 nb_rx = 0; 998 if (unlikely(hw->started == 0)) 999 return nb_rx; 1000 1001 nb_used = virtqueue_nused(vq); 1002 1003 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1004 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1005 num = VIRTIO_MBUF_BURST_SZ; 1006 if (likely(num > DESC_PER_CACHELINE)) 1007 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1008 1009 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1010 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 1011 1012 nb_enqueued = 0; 1013 hdr_size = hw->vtnet_hdr_size; 1014 1015 for (i = 0; i < num ; i++) { 1016 rxm = rcv_pkts[i]; 1017 1018 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1019 1020 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1021 PMD_RX_LOG(ERR, "Packet drop"); 1022 nb_enqueued++; 1023 virtio_discard_rxbuf(vq, rxm); 1024 rxvq->stats.errors++; 1025 continue; 1026 } 1027 1028 rxm->port = rxvq->port_id; 1029 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1030 rxm->ol_flags = 0; 1031 rxm->vlan_tci = 0; 1032 1033 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1034 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1035 1036 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1037 RTE_PKTMBUF_HEADROOM - hdr_size); 1038 1039 if (hw->vlan_strip) 1040 rte_vlan_strip(rxm); 1041 1042 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1043 virtio_discard_rxbuf(vq, rxm); 1044 rxvq->stats.errors++; 1045 continue; 1046 } 1047 1048 virtio_rx_stats_updated(rxvq, rxm); 1049 1050 rx_pkts[nb_rx++] = rxm; 1051 } 1052 1053 rxvq->stats.packets += nb_rx; 1054 1055 /* Allocate new mbuf for the used descriptor */ 1056 if (likely(!virtqueue_full(vq))) { 1057 uint16_t free_cnt = vq->vq_free_cnt; 1058 struct rte_mbuf *new_pkts[free_cnt]; 1059 1060 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1061 free_cnt) == 0)) { 1062 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1063 free_cnt); 1064 if (unlikely(error)) { 1065 for (i = 0; i < free_cnt; i++) 1066 rte_pktmbuf_free(new_pkts[i]); 1067 } 1068 nb_enqueued += free_cnt; 1069 } else { 1070 struct rte_eth_dev *dev = 1071 &rte_eth_devices[rxvq->port_id]; 1072 dev->data->rx_mbuf_alloc_failed += free_cnt; 1073 } 1074 } 1075 1076 if (likely(nb_enqueued)) { 1077 vq_update_avail_idx(vq); 1078 1079 if (unlikely(virtqueue_kick_prepare(vq))) { 1080 virtqueue_notify(vq); 1081 PMD_RX_LOG(DEBUG, "Notified"); 1082 } 1083 } 1084 1085 return nb_rx; 1086 } 1087 1088 uint16_t 1089 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1090 uint16_t nb_pkts) 1091 { 1092 struct virtnet_rx *rxvq = rx_queue; 1093 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1094 struct virtio_hw *hw = vq->hw; 1095 struct rte_mbuf *rxm; 1096 uint16_t num, nb_rx; 1097 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1098 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1099 int error; 1100 uint32_t i, nb_enqueued; 1101 uint32_t hdr_size; 1102 struct virtio_net_hdr *hdr; 1103 1104 nb_rx = 0; 1105 if (unlikely(hw->started == 0)) 1106 return nb_rx; 1107 1108 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1109 if (likely(num > DESC_PER_CACHELINE)) 1110 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1111 1112 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1113 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1114 1115 nb_enqueued = 0; 1116 hdr_size = hw->vtnet_hdr_size; 1117 1118 for (i = 0; i < num; i++) { 1119 rxm = rcv_pkts[i]; 1120 1121 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1122 1123 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1124 PMD_RX_LOG(ERR, "Packet drop"); 1125 nb_enqueued++; 1126 virtio_discard_rxbuf(vq, rxm); 1127 rxvq->stats.errors++; 1128 continue; 1129 } 1130 1131 rxm->port = rxvq->port_id; 1132 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1133 rxm->ol_flags = 0; 1134 rxm->vlan_tci = 0; 1135 1136 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1137 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1138 1139 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1140 RTE_PKTMBUF_HEADROOM - hdr_size); 1141 1142 if (hw->vlan_strip) 1143 rte_vlan_strip(rxm); 1144 1145 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1146 virtio_discard_rxbuf(vq, rxm); 1147 rxvq->stats.errors++; 1148 continue; 1149 } 1150 1151 virtio_rx_stats_updated(rxvq, rxm); 1152 1153 rx_pkts[nb_rx++] = rxm; 1154 } 1155 1156 rxvq->stats.packets += nb_rx; 1157 1158 /* Allocate new mbuf for the used descriptor */ 1159 if (likely(!virtqueue_full(vq))) { 1160 uint16_t free_cnt = vq->vq_free_cnt; 1161 struct rte_mbuf *new_pkts[free_cnt]; 1162 1163 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1164 free_cnt) == 0)) { 1165 error = virtqueue_enqueue_recv_refill_packed(vq, 1166 new_pkts, free_cnt); 1167 if (unlikely(error)) { 1168 for (i = 0; i < free_cnt; i++) 1169 rte_pktmbuf_free(new_pkts[i]); 1170 } 1171 nb_enqueued += free_cnt; 1172 } else { 1173 struct rte_eth_dev *dev = 1174 &rte_eth_devices[rxvq->port_id]; 1175 dev->data->rx_mbuf_alloc_failed += free_cnt; 1176 } 1177 } 1178 1179 if (likely(nb_enqueued)) { 1180 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1181 virtqueue_notify(vq); 1182 PMD_RX_LOG(DEBUG, "Notified"); 1183 } 1184 } 1185 1186 return nb_rx; 1187 } 1188 1189 1190 uint16_t 1191 virtio_recv_pkts_inorder(void *rx_queue, 1192 struct rte_mbuf **rx_pkts, 1193 uint16_t nb_pkts) 1194 { 1195 struct virtnet_rx *rxvq = rx_queue; 1196 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1197 struct virtio_hw *hw = vq->hw; 1198 struct rte_mbuf *rxm; 1199 struct rte_mbuf *prev = NULL; 1200 uint16_t nb_used, num, nb_rx; 1201 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1202 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1203 int error; 1204 uint32_t nb_enqueued; 1205 uint32_t seg_num; 1206 uint32_t seg_res; 1207 uint32_t hdr_size; 1208 int32_t i; 1209 1210 nb_rx = 0; 1211 if (unlikely(hw->started == 0)) 1212 return nb_rx; 1213 1214 nb_used = virtqueue_nused(vq); 1215 nb_used = RTE_MIN(nb_used, nb_pkts); 1216 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1217 1218 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1219 1220 nb_enqueued = 0; 1221 seg_num = 1; 1222 seg_res = 0; 1223 hdr_size = hw->vtnet_hdr_size; 1224 1225 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1226 1227 for (i = 0; i < num; i++) { 1228 struct virtio_net_hdr_mrg_rxbuf *header; 1229 1230 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1231 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1232 1233 rxm = rcv_pkts[i]; 1234 1235 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1236 PMD_RX_LOG(ERR, "Packet drop"); 1237 nb_enqueued++; 1238 virtio_discard_rxbuf_inorder(vq, rxm); 1239 rxvq->stats.errors++; 1240 continue; 1241 } 1242 1243 header = (struct virtio_net_hdr_mrg_rxbuf *) 1244 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1245 - hdr_size); 1246 1247 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1248 seg_num = header->num_buffers; 1249 if (seg_num == 0) 1250 seg_num = 1; 1251 } else { 1252 seg_num = 1; 1253 } 1254 1255 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1256 rxm->nb_segs = seg_num; 1257 rxm->ol_flags = 0; 1258 rxm->vlan_tci = 0; 1259 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1260 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1261 1262 rxm->port = rxvq->port_id; 1263 1264 rx_pkts[nb_rx] = rxm; 1265 prev = rxm; 1266 1267 if (vq->hw->has_rx_offload && 1268 virtio_rx_offload(rxm, &header->hdr) < 0) { 1269 virtio_discard_rxbuf_inorder(vq, rxm); 1270 rxvq->stats.errors++; 1271 continue; 1272 } 1273 1274 if (hw->vlan_strip) 1275 rte_vlan_strip(rx_pkts[nb_rx]); 1276 1277 seg_res = seg_num - 1; 1278 1279 /* Merge remaining segments */ 1280 while (seg_res != 0 && i < (num - 1)) { 1281 i++; 1282 1283 rxm = rcv_pkts[i]; 1284 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1285 rxm->pkt_len = (uint32_t)(len[i]); 1286 rxm->data_len = (uint16_t)(len[i]); 1287 1288 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1289 1290 prev->next = rxm; 1291 prev = rxm; 1292 seg_res -= 1; 1293 } 1294 1295 if (!seg_res) { 1296 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1297 nb_rx++; 1298 } 1299 } 1300 1301 /* Last packet still need merge segments */ 1302 while (seg_res != 0) { 1303 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1304 VIRTIO_MBUF_BURST_SZ); 1305 1306 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1307 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1308 rcv_cnt); 1309 uint16_t extra_idx = 0; 1310 1311 rcv_cnt = num; 1312 while (extra_idx < rcv_cnt) { 1313 rxm = rcv_pkts[extra_idx]; 1314 rxm->data_off = 1315 RTE_PKTMBUF_HEADROOM - hdr_size; 1316 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1317 rxm->data_len = (uint16_t)(len[extra_idx]); 1318 prev->next = rxm; 1319 prev = rxm; 1320 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1321 extra_idx += 1; 1322 }; 1323 seg_res -= rcv_cnt; 1324 1325 if (!seg_res) { 1326 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1327 nb_rx++; 1328 } 1329 } else { 1330 PMD_RX_LOG(ERR, 1331 "No enough segments for packet."); 1332 rte_pktmbuf_free(rx_pkts[nb_rx]); 1333 rxvq->stats.errors++; 1334 break; 1335 } 1336 } 1337 1338 rxvq->stats.packets += nb_rx; 1339 1340 /* Allocate new mbuf for the used descriptor */ 1341 1342 if (likely(!virtqueue_full(vq))) { 1343 /* free_cnt may include mrg descs */ 1344 uint16_t free_cnt = vq->vq_free_cnt; 1345 struct rte_mbuf *new_pkts[free_cnt]; 1346 1347 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1348 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1349 free_cnt); 1350 if (unlikely(error)) { 1351 for (i = 0; i < free_cnt; i++) 1352 rte_pktmbuf_free(new_pkts[i]); 1353 } 1354 nb_enqueued += free_cnt; 1355 } else { 1356 struct rte_eth_dev *dev = 1357 &rte_eth_devices[rxvq->port_id]; 1358 dev->data->rx_mbuf_alloc_failed += free_cnt; 1359 } 1360 } 1361 1362 if (likely(nb_enqueued)) { 1363 vq_update_avail_idx(vq); 1364 1365 if (unlikely(virtqueue_kick_prepare(vq))) { 1366 virtqueue_notify(vq); 1367 PMD_RX_LOG(DEBUG, "Notified"); 1368 } 1369 } 1370 1371 return nb_rx; 1372 } 1373 1374 uint16_t 1375 virtio_recv_mergeable_pkts(void *rx_queue, 1376 struct rte_mbuf **rx_pkts, 1377 uint16_t nb_pkts) 1378 { 1379 struct virtnet_rx *rxvq = rx_queue; 1380 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1381 struct virtio_hw *hw = vq->hw; 1382 struct rte_mbuf *rxm; 1383 struct rte_mbuf *prev = NULL; 1384 uint16_t nb_used, num, nb_rx = 0; 1385 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1386 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1387 int error; 1388 uint32_t nb_enqueued = 0; 1389 uint32_t seg_num = 0; 1390 uint32_t seg_res = 0; 1391 uint32_t hdr_size = hw->vtnet_hdr_size; 1392 int32_t i; 1393 1394 if (unlikely(hw->started == 0)) 1395 return nb_rx; 1396 1397 nb_used = virtqueue_nused(vq); 1398 1399 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1400 1401 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1402 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1403 num = VIRTIO_MBUF_BURST_SZ; 1404 if (likely(num > DESC_PER_CACHELINE)) 1405 num = num - ((vq->vq_used_cons_idx + num) % 1406 DESC_PER_CACHELINE); 1407 1408 1409 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1410 1411 for (i = 0; i < num; i++) { 1412 struct virtio_net_hdr_mrg_rxbuf *header; 1413 1414 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1415 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1416 1417 rxm = rcv_pkts[i]; 1418 1419 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1420 PMD_RX_LOG(ERR, "Packet drop"); 1421 nb_enqueued++; 1422 virtio_discard_rxbuf(vq, rxm); 1423 rxvq->stats.errors++; 1424 continue; 1425 } 1426 1427 header = (struct virtio_net_hdr_mrg_rxbuf *) 1428 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1429 - hdr_size); 1430 seg_num = header->num_buffers; 1431 if (seg_num == 0) 1432 seg_num = 1; 1433 1434 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1435 rxm->nb_segs = seg_num; 1436 rxm->ol_flags = 0; 1437 rxm->vlan_tci = 0; 1438 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1439 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1440 1441 rxm->port = rxvq->port_id; 1442 1443 rx_pkts[nb_rx] = rxm; 1444 prev = rxm; 1445 1446 if (hw->has_rx_offload && 1447 virtio_rx_offload(rxm, &header->hdr) < 0) { 1448 virtio_discard_rxbuf(vq, rxm); 1449 rxvq->stats.errors++; 1450 continue; 1451 } 1452 1453 if (hw->vlan_strip) 1454 rte_vlan_strip(rx_pkts[nb_rx]); 1455 1456 seg_res = seg_num - 1; 1457 1458 /* Merge remaining segments */ 1459 while (seg_res != 0 && i < (num - 1)) { 1460 i++; 1461 1462 rxm = rcv_pkts[i]; 1463 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1464 rxm->pkt_len = (uint32_t)(len[i]); 1465 rxm->data_len = (uint16_t)(len[i]); 1466 1467 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1468 1469 prev->next = rxm; 1470 prev = rxm; 1471 seg_res -= 1; 1472 } 1473 1474 if (!seg_res) { 1475 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1476 nb_rx++; 1477 } 1478 } 1479 1480 /* Last packet still need merge segments */ 1481 while (seg_res != 0) { 1482 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1483 VIRTIO_MBUF_BURST_SZ); 1484 1485 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1486 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1487 rcv_cnt); 1488 uint16_t extra_idx = 0; 1489 1490 rcv_cnt = num; 1491 while (extra_idx < rcv_cnt) { 1492 rxm = rcv_pkts[extra_idx]; 1493 rxm->data_off = 1494 RTE_PKTMBUF_HEADROOM - hdr_size; 1495 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1496 rxm->data_len = (uint16_t)(len[extra_idx]); 1497 prev->next = rxm; 1498 prev = rxm; 1499 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1500 extra_idx += 1; 1501 }; 1502 seg_res -= rcv_cnt; 1503 1504 if (!seg_res) { 1505 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1506 nb_rx++; 1507 } 1508 } else { 1509 PMD_RX_LOG(ERR, 1510 "No enough segments for packet."); 1511 rte_pktmbuf_free(rx_pkts[nb_rx]); 1512 rxvq->stats.errors++; 1513 break; 1514 } 1515 } 1516 1517 rxvq->stats.packets += nb_rx; 1518 1519 /* Allocate new mbuf for the used descriptor */ 1520 if (likely(!virtqueue_full(vq))) { 1521 /* free_cnt may include mrg descs */ 1522 uint16_t free_cnt = vq->vq_free_cnt; 1523 struct rte_mbuf *new_pkts[free_cnt]; 1524 1525 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1526 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1527 free_cnt); 1528 if (unlikely(error)) { 1529 for (i = 0; i < free_cnt; i++) 1530 rte_pktmbuf_free(new_pkts[i]); 1531 } 1532 nb_enqueued += free_cnt; 1533 } else { 1534 struct rte_eth_dev *dev = 1535 &rte_eth_devices[rxvq->port_id]; 1536 dev->data->rx_mbuf_alloc_failed += free_cnt; 1537 } 1538 } 1539 1540 if (likely(nb_enqueued)) { 1541 vq_update_avail_idx(vq); 1542 1543 if (unlikely(virtqueue_kick_prepare(vq))) { 1544 virtqueue_notify(vq); 1545 PMD_RX_LOG(DEBUG, "Notified"); 1546 } 1547 } 1548 1549 return nb_rx; 1550 } 1551 1552 uint16_t 1553 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1554 struct rte_mbuf **rx_pkts, 1555 uint16_t nb_pkts) 1556 { 1557 struct virtnet_rx *rxvq = rx_queue; 1558 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1559 struct virtio_hw *hw = vq->hw; 1560 struct rte_mbuf *rxm; 1561 struct rte_mbuf *prev = NULL; 1562 uint16_t num, nb_rx = 0; 1563 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1564 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1565 uint32_t nb_enqueued = 0; 1566 uint32_t seg_num = 0; 1567 uint32_t seg_res = 0; 1568 uint32_t hdr_size = hw->vtnet_hdr_size; 1569 int32_t i; 1570 int error; 1571 1572 if (unlikely(hw->started == 0)) 1573 return nb_rx; 1574 1575 1576 num = nb_pkts; 1577 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1578 num = VIRTIO_MBUF_BURST_SZ; 1579 if (likely(num > DESC_PER_CACHELINE)) 1580 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1581 1582 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1583 1584 for (i = 0; i < num; i++) { 1585 struct virtio_net_hdr_mrg_rxbuf *header; 1586 1587 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1588 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1589 1590 rxm = rcv_pkts[i]; 1591 1592 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1593 PMD_RX_LOG(ERR, "Packet drop"); 1594 nb_enqueued++; 1595 virtio_discard_rxbuf(vq, rxm); 1596 rxvq->stats.errors++; 1597 continue; 1598 } 1599 1600 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1601 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1602 seg_num = header->num_buffers; 1603 1604 if (seg_num == 0) 1605 seg_num = 1; 1606 1607 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1608 rxm->nb_segs = seg_num; 1609 rxm->ol_flags = 0; 1610 rxm->vlan_tci = 0; 1611 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1612 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1613 1614 rxm->port = rxvq->port_id; 1615 rx_pkts[nb_rx] = rxm; 1616 prev = rxm; 1617 1618 if (hw->has_rx_offload && 1619 virtio_rx_offload(rxm, &header->hdr) < 0) { 1620 virtio_discard_rxbuf(vq, rxm); 1621 rxvq->stats.errors++; 1622 continue; 1623 } 1624 1625 if (hw->vlan_strip) 1626 rte_vlan_strip(rx_pkts[nb_rx]); 1627 1628 seg_res = seg_num - 1; 1629 1630 /* Merge remaining segments */ 1631 while (seg_res != 0 && i < (num - 1)) { 1632 i++; 1633 1634 rxm = rcv_pkts[i]; 1635 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1636 rxm->pkt_len = (uint32_t)(len[i]); 1637 rxm->data_len = (uint16_t)(len[i]); 1638 1639 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1640 1641 prev->next = rxm; 1642 prev = rxm; 1643 seg_res -= 1; 1644 } 1645 1646 if (!seg_res) { 1647 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1648 nb_rx++; 1649 } 1650 } 1651 1652 /* Last packet still need merge segments */ 1653 while (seg_res != 0) { 1654 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1655 VIRTIO_MBUF_BURST_SZ); 1656 uint16_t extra_idx = 0; 1657 1658 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1659 len, rcv_cnt); 1660 if (unlikely(rcv_cnt == 0)) { 1661 PMD_RX_LOG(ERR, "No enough segments for packet."); 1662 rte_pktmbuf_free(rx_pkts[nb_rx]); 1663 rxvq->stats.errors++; 1664 break; 1665 } 1666 1667 while (extra_idx < rcv_cnt) { 1668 rxm = rcv_pkts[extra_idx]; 1669 1670 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1671 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1672 rxm->data_len = (uint16_t)(len[extra_idx]); 1673 1674 prev->next = rxm; 1675 prev = rxm; 1676 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1677 extra_idx += 1; 1678 } 1679 seg_res -= rcv_cnt; 1680 if (!seg_res) { 1681 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1682 nb_rx++; 1683 } 1684 } 1685 1686 rxvq->stats.packets += nb_rx; 1687 1688 /* Allocate new mbuf for the used descriptor */ 1689 if (likely(!virtqueue_full(vq))) { 1690 /* free_cnt may include mrg descs */ 1691 uint16_t free_cnt = vq->vq_free_cnt; 1692 struct rte_mbuf *new_pkts[free_cnt]; 1693 1694 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1695 error = virtqueue_enqueue_recv_refill_packed(vq, 1696 new_pkts, free_cnt); 1697 if (unlikely(error)) { 1698 for (i = 0; i < free_cnt; i++) 1699 rte_pktmbuf_free(new_pkts[i]); 1700 } 1701 nb_enqueued += free_cnt; 1702 } else { 1703 struct rte_eth_dev *dev = 1704 &rte_eth_devices[rxvq->port_id]; 1705 dev->data->rx_mbuf_alloc_failed += free_cnt; 1706 } 1707 } 1708 1709 if (likely(nb_enqueued)) { 1710 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1711 virtqueue_notify(vq); 1712 PMD_RX_LOG(DEBUG, "Notified"); 1713 } 1714 } 1715 1716 return nb_rx; 1717 } 1718 1719 uint16_t 1720 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1721 uint16_t nb_pkts) 1722 { 1723 uint16_t nb_tx; 1724 int error; 1725 1726 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1727 struct rte_mbuf *m = tx_pkts[nb_tx]; 1728 1729 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1730 error = rte_validate_tx_offload(m); 1731 if (unlikely(error)) { 1732 rte_errno = -error; 1733 break; 1734 } 1735 #endif 1736 1737 /* Do VLAN tag insertion */ 1738 if (unlikely(m->ol_flags & RTE_MBUF_F_TX_VLAN)) { 1739 error = rte_vlan_insert(&m); 1740 /* rte_vlan_insert() may change pointer 1741 * even in the case of failure 1742 */ 1743 tx_pkts[nb_tx] = m; 1744 1745 if (unlikely(error)) { 1746 rte_errno = -error; 1747 break; 1748 } 1749 } 1750 1751 error = rte_net_intel_cksum_prepare(m); 1752 if (unlikely(error)) { 1753 rte_errno = -error; 1754 break; 1755 } 1756 1757 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1758 virtio_tso_fix_cksum(m); 1759 } 1760 1761 return nb_tx; 1762 } 1763 1764 uint16_t 1765 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 1766 uint16_t nb_pkts) 1767 { 1768 struct virtnet_tx *txvq = tx_queue; 1769 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1770 struct virtio_hw *hw = vq->hw; 1771 uint16_t hdr_size = hw->vtnet_hdr_size; 1772 uint16_t nb_tx = 0; 1773 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 1774 1775 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1776 return nb_tx; 1777 1778 if (unlikely(nb_pkts < 1)) 1779 return nb_pkts; 1780 1781 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1782 1783 if (nb_pkts > vq->vq_free_cnt) 1784 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 1785 in_order); 1786 1787 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1788 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1789 int can_push = 0, use_indirect = 0, slots, need; 1790 1791 /* optimize ring usage */ 1792 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1793 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1794 rte_mbuf_refcnt_read(txm) == 1 && 1795 RTE_MBUF_DIRECT(txm) && 1796 txm->nb_segs == 1 && 1797 rte_pktmbuf_headroom(txm) >= hdr_size && 1798 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1799 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1800 can_push = 1; 1801 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1802 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1803 use_indirect = 1; 1804 /* How many main ring entries are needed to this Tx? 1805 * indirect => 1 1806 * any_layout => number of segments 1807 * default => number of segments + 1 1808 */ 1809 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1810 need = slots - vq->vq_free_cnt; 1811 1812 /* Positive value indicates it need free vring descriptors */ 1813 if (unlikely(need > 0)) { 1814 virtio_xmit_cleanup_packed(vq, need, in_order); 1815 need = slots - vq->vq_free_cnt; 1816 if (unlikely(need > 0)) { 1817 PMD_TX_LOG(ERR, 1818 "No free tx descriptors to transmit"); 1819 break; 1820 } 1821 } 1822 1823 /* Enqueue Packet buffers */ 1824 if (can_push) 1825 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 1826 else 1827 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 1828 use_indirect, 0, 1829 in_order); 1830 1831 virtio_update_packet_stats(&txvq->stats, txm); 1832 } 1833 1834 txvq->stats.packets += nb_tx; 1835 1836 if (likely(nb_tx)) { 1837 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1838 virtqueue_notify(vq); 1839 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1840 } 1841 } 1842 1843 return nb_tx; 1844 } 1845 1846 uint16_t 1847 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1848 { 1849 struct virtnet_tx *txvq = tx_queue; 1850 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1851 struct virtio_hw *hw = vq->hw; 1852 uint16_t hdr_size = hw->vtnet_hdr_size; 1853 uint16_t nb_used, nb_tx = 0; 1854 1855 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1856 return nb_tx; 1857 1858 if (unlikely(nb_pkts < 1)) 1859 return nb_pkts; 1860 1861 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1862 1863 nb_used = virtqueue_nused(vq); 1864 1865 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1866 virtio_xmit_cleanup(vq, nb_used); 1867 1868 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1869 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1870 int can_push = 0, use_indirect = 0, slots, need; 1871 1872 /* optimize ring usage */ 1873 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1874 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1875 rte_mbuf_refcnt_read(txm) == 1 && 1876 RTE_MBUF_DIRECT(txm) && 1877 txm->nb_segs == 1 && 1878 rte_pktmbuf_headroom(txm) >= hdr_size && 1879 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1880 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1881 can_push = 1; 1882 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1883 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1884 use_indirect = 1; 1885 1886 /* How many main ring entries are needed to this Tx? 1887 * any_layout => number of segments 1888 * indirect => 1 1889 * default => number of segments + 1 1890 */ 1891 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1892 need = slots - vq->vq_free_cnt; 1893 1894 /* Positive value indicates it need free vring descriptors */ 1895 if (unlikely(need > 0)) { 1896 nb_used = virtqueue_nused(vq); 1897 1898 need = RTE_MIN(need, (int)nb_used); 1899 1900 virtio_xmit_cleanup(vq, need); 1901 need = slots - vq->vq_free_cnt; 1902 if (unlikely(need > 0)) { 1903 PMD_TX_LOG(ERR, 1904 "No free tx descriptors to transmit"); 1905 break; 1906 } 1907 } 1908 1909 /* Enqueue Packet buffers */ 1910 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 1911 can_push, 0); 1912 1913 virtio_update_packet_stats(&txvq->stats, txm); 1914 } 1915 1916 txvq->stats.packets += nb_tx; 1917 1918 if (likely(nb_tx)) { 1919 vq_update_avail_idx(vq); 1920 1921 if (unlikely(virtqueue_kick_prepare(vq))) { 1922 virtqueue_notify(vq); 1923 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1924 } 1925 } 1926 1927 return nb_tx; 1928 } 1929 1930 static __rte_always_inline int 1931 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 1932 { 1933 uint16_t nb_used, nb_clean, nb_descs; 1934 1935 nb_descs = vq->vq_free_cnt + need; 1936 nb_used = virtqueue_nused(vq); 1937 nb_clean = RTE_MIN(need, (int)nb_used); 1938 1939 virtio_xmit_cleanup_inorder(vq, nb_clean); 1940 1941 return nb_descs - vq->vq_free_cnt; 1942 } 1943 1944 uint16_t 1945 virtio_xmit_pkts_inorder(void *tx_queue, 1946 struct rte_mbuf **tx_pkts, 1947 uint16_t nb_pkts) 1948 { 1949 struct virtnet_tx *txvq = tx_queue; 1950 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1951 struct virtio_hw *hw = vq->hw; 1952 uint16_t hdr_size = hw->vtnet_hdr_size; 1953 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 1954 struct rte_mbuf *inorder_pkts[nb_pkts]; 1955 int need; 1956 1957 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1958 return nb_tx; 1959 1960 if (unlikely(nb_pkts < 1)) 1961 return nb_pkts; 1962 1963 VIRTQUEUE_DUMP(vq); 1964 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1965 nb_used = virtqueue_nused(vq); 1966 1967 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1968 virtio_xmit_cleanup_inorder(vq, nb_used); 1969 1970 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1971 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1972 int slots; 1973 1974 /* optimize ring usage */ 1975 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1976 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1977 rte_mbuf_refcnt_read(txm) == 1 && 1978 RTE_MBUF_DIRECT(txm) && 1979 txm->nb_segs == 1 && 1980 rte_pktmbuf_headroom(txm) >= hdr_size && 1981 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1982 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) { 1983 inorder_pkts[nb_inorder_pkts] = txm; 1984 nb_inorder_pkts++; 1985 1986 continue; 1987 } 1988 1989 if (nb_inorder_pkts) { 1990 need = nb_inorder_pkts - vq->vq_free_cnt; 1991 if (unlikely(need > 0)) { 1992 need = virtio_xmit_try_cleanup_inorder(vq, 1993 need); 1994 if (unlikely(need > 0)) { 1995 PMD_TX_LOG(ERR, 1996 "No free tx descriptors to " 1997 "transmit"); 1998 break; 1999 } 2000 } 2001 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2002 nb_inorder_pkts); 2003 nb_inorder_pkts = 0; 2004 } 2005 2006 slots = txm->nb_segs + 1; 2007 need = slots - vq->vq_free_cnt; 2008 if (unlikely(need > 0)) { 2009 need = virtio_xmit_try_cleanup_inorder(vq, slots); 2010 2011 if (unlikely(need > 0)) { 2012 PMD_TX_LOG(ERR, 2013 "No free tx descriptors to transmit"); 2014 break; 2015 } 2016 } 2017 /* Enqueue Packet buffers */ 2018 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 2019 2020 virtio_update_packet_stats(&txvq->stats, txm); 2021 } 2022 2023 /* Transmit all inorder packets */ 2024 if (nb_inorder_pkts) { 2025 need = nb_inorder_pkts - vq->vq_free_cnt; 2026 if (unlikely(need > 0)) { 2027 need = virtio_xmit_try_cleanup_inorder(vq, 2028 need); 2029 if (unlikely(need > 0)) { 2030 PMD_TX_LOG(ERR, 2031 "No free tx descriptors to transmit"); 2032 nb_inorder_pkts = vq->vq_free_cnt; 2033 nb_tx -= need; 2034 } 2035 } 2036 2037 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2038 nb_inorder_pkts); 2039 } 2040 2041 txvq->stats.packets += nb_tx; 2042 2043 if (likely(nb_tx)) { 2044 vq_update_avail_idx(vq); 2045 2046 if (unlikely(virtqueue_kick_prepare(vq))) { 2047 virtqueue_notify(vq); 2048 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2049 } 2050 } 2051 2052 VIRTQUEUE_DUMP(vq); 2053 2054 return nb_tx; 2055 } 2056 2057 __rte_weak uint16_t 2058 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, 2059 struct rte_mbuf **rx_pkts __rte_unused, 2060 uint16_t nb_pkts __rte_unused) 2061 { 2062 return 0; 2063 } 2064 2065 __rte_weak uint16_t 2066 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, 2067 struct rte_mbuf **tx_pkts __rte_unused, 2068 uint16_t nb_pkts __rte_unused) 2069 { 2070 return 0; 2071 } 2072