1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_cycles.h> 12 #include <rte_memory.h> 13 #include <rte_branch_prediction.h> 14 #include <rte_mempool.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_ether.h> 18 #include <ethdev_driver.h> 19 #include <rte_prefetch.h> 20 #include <rte_string_fns.h> 21 #include <rte_errno.h> 22 #include <rte_byteorder.h> 23 #include <rte_net.h> 24 #include <rte_ip.h> 25 #include <rte_udp.h> 26 #include <rte_tcp.h> 27 28 #include "virtio_logs.h" 29 #include "virtio_ethdev.h" 30 #include "virtio.h" 31 #include "virtqueue.h" 32 #include "virtio_rxtx.h" 33 #include "virtio_rxtx_simple.h" 34 #include "virtio_ring.h" 35 36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 38 #else 39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 40 #endif 41 42 void 43 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 44 { 45 vq->vq_free_cnt += num; 46 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 47 } 48 49 void 50 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 51 { 52 struct vring_desc *dp, *dp_tail; 53 struct vq_desc_extra *dxp; 54 uint16_t desc_idx_last = desc_idx; 55 56 dp = &vq->vq_split.ring.desc[desc_idx]; 57 dxp = &vq->vq_descx[desc_idx]; 58 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 59 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 60 while (dp->flags & VRING_DESC_F_NEXT) { 61 desc_idx_last = dp->next; 62 dp = &vq->vq_split.ring.desc[dp->next]; 63 } 64 } 65 dxp->ndescs = 0; 66 67 /* 68 * We must append the existing free chain, if any, to the end of 69 * newly freed chain. If the virtqueue was completely used, then 70 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 71 */ 72 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 73 vq->vq_desc_head_idx = desc_idx; 74 } else { 75 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 76 dp_tail->next = desc_idx; 77 } 78 79 vq->vq_desc_tail_idx = desc_idx_last; 80 dp->next = VQ_RING_DESC_CHAIN_END; 81 } 82 83 void 84 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) 85 { 86 uint32_t s = mbuf->pkt_len; 87 struct rte_ether_addr *ea; 88 89 stats->bytes += s; 90 91 if (s == 64) { 92 stats->size_bins[1]++; 93 } else if (s > 64 && s < 1024) { 94 uint32_t bin; 95 96 /* count zeros, and offset into correct bin */ 97 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 98 stats->size_bins[bin]++; 99 } else { 100 if (s < 64) 101 stats->size_bins[0]++; 102 else if (s < 1519) 103 stats->size_bins[6]++; 104 else 105 stats->size_bins[7]++; 106 } 107 108 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 109 if (rte_is_multicast_ether_addr(ea)) { 110 if (rte_is_broadcast_ether_addr(ea)) 111 stats->broadcast++; 112 else 113 stats->multicast++; 114 } 115 } 116 117 static inline void 118 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 119 { 120 VIRTIO_DUMP_PACKET(m, m->data_len); 121 122 virtio_update_packet_stats(&rxvq->stats, m); 123 } 124 125 static uint16_t 126 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 127 struct rte_mbuf **rx_pkts, 128 uint32_t *len, 129 uint16_t num) 130 { 131 struct rte_mbuf *cookie; 132 uint16_t used_idx; 133 uint16_t id; 134 struct vring_packed_desc *desc; 135 uint16_t i; 136 137 desc = vq->vq_packed.ring.desc; 138 139 for (i = 0; i < num; i++) { 140 used_idx = vq->vq_used_cons_idx; 141 /* desc_is_used has a load-acquire or rte_io_rmb inside 142 * and wait for used desc in virtqueue. 143 */ 144 if (!desc_is_used(&desc[used_idx], vq)) 145 return i; 146 len[i] = desc[used_idx].len; 147 id = desc[used_idx].id; 148 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 149 if (unlikely(cookie == NULL)) { 150 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 151 vq->vq_used_cons_idx); 152 break; 153 } 154 rte_prefetch0(cookie); 155 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 156 rx_pkts[i] = cookie; 157 158 vq->vq_free_cnt++; 159 vq->vq_used_cons_idx++; 160 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 161 vq->vq_used_cons_idx -= vq->vq_nentries; 162 vq->vq_packed.used_wrap_counter ^= 1; 163 } 164 } 165 166 return i; 167 } 168 169 static uint16_t 170 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 171 uint32_t *len, uint16_t num) 172 { 173 struct vring_used_elem *uep; 174 struct rte_mbuf *cookie; 175 uint16_t used_idx, desc_idx; 176 uint16_t i; 177 178 /* Caller does the check */ 179 for (i = 0; i < num ; i++) { 180 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 181 uep = &vq->vq_split.ring.used->ring[used_idx]; 182 desc_idx = (uint16_t) uep->id; 183 len[i] = uep->len; 184 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 185 186 if (unlikely(cookie == NULL)) { 187 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 188 vq->vq_used_cons_idx); 189 break; 190 } 191 192 rte_prefetch0(cookie); 193 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 194 rx_pkts[i] = cookie; 195 vq->vq_used_cons_idx++; 196 vq_ring_free_chain(vq, desc_idx); 197 vq->vq_descx[desc_idx].cookie = NULL; 198 } 199 200 return i; 201 } 202 203 static uint16_t 204 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 205 struct rte_mbuf **rx_pkts, 206 uint32_t *len, 207 uint16_t num) 208 { 209 struct vring_used_elem *uep; 210 struct rte_mbuf *cookie; 211 uint16_t used_idx = 0; 212 uint16_t i; 213 214 if (unlikely(num == 0)) 215 return 0; 216 217 for (i = 0; i < num; i++) { 218 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 219 /* Desc idx same as used idx */ 220 uep = &vq->vq_split.ring.used->ring[used_idx]; 221 len[i] = uep->len; 222 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 223 224 if (unlikely(cookie == NULL)) { 225 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 226 vq->vq_used_cons_idx); 227 break; 228 } 229 230 rte_prefetch0(cookie); 231 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 232 rx_pkts[i] = cookie; 233 vq->vq_used_cons_idx++; 234 vq->vq_descx[used_idx].cookie = NULL; 235 } 236 237 vq_ring_free_inorder(vq, used_idx, i); 238 return i; 239 } 240 241 static inline int 242 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 243 struct rte_mbuf **cookies, 244 uint16_t num) 245 { 246 struct vq_desc_extra *dxp; 247 struct virtio_hw *hw = vq->hw; 248 struct vring_desc *start_dp; 249 uint16_t head_idx, idx, i = 0; 250 251 if (unlikely(vq->vq_free_cnt == 0)) 252 return -ENOSPC; 253 if (unlikely(vq->vq_free_cnt < num)) 254 return -EMSGSIZE; 255 256 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 257 start_dp = vq->vq_split.ring.desc; 258 259 while (i < num) { 260 idx = head_idx & (vq->vq_nentries - 1); 261 dxp = &vq->vq_descx[idx]; 262 dxp->cookie = (void *)cookies[i]; 263 dxp->ndescs = 1; 264 265 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) + 266 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 267 start_dp[idx].len = cookies[i]->buf_len - 268 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 269 start_dp[idx].flags = VRING_DESC_F_WRITE; 270 271 vq_update_avail_ring(vq, idx); 272 head_idx++; 273 i++; 274 } 275 276 vq->vq_desc_head_idx += num; 277 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 278 return 0; 279 } 280 281 static inline int 282 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 283 uint16_t num) 284 { 285 struct vq_desc_extra *dxp; 286 struct virtio_hw *hw = vq->hw; 287 struct vring_desc *start_dp = vq->vq_split.ring.desc; 288 uint16_t idx, i; 289 290 if (unlikely(vq->vq_free_cnt == 0)) 291 return -ENOSPC; 292 if (unlikely(vq->vq_free_cnt < num)) 293 return -EMSGSIZE; 294 295 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 296 return -EFAULT; 297 298 for (i = 0; i < num; i++) { 299 idx = vq->vq_desc_head_idx; 300 dxp = &vq->vq_descx[idx]; 301 dxp->cookie = (void *)cookie[i]; 302 dxp->ndescs = 1; 303 304 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + 305 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 306 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + 307 hw->vtnet_hdr_size; 308 start_dp[idx].flags = VRING_DESC_F_WRITE; 309 vq->vq_desc_head_idx = start_dp[idx].next; 310 vq_update_avail_ring(vq, idx); 311 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 312 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 313 break; 314 } 315 } 316 317 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 318 319 return 0; 320 } 321 322 static inline void 323 virtqueue_refill_single_packed(struct virtqueue *vq, 324 struct vring_packed_desc *dp, 325 struct rte_mbuf *cookie) 326 { 327 uint16_t flags = vq->vq_packed.cached_flags; 328 struct virtio_hw *hw = vq->hw; 329 330 dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 331 dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 332 333 virtqueue_store_flags_packed(dp, flags, hw->weak_barriers); 334 335 if (++vq->vq_avail_idx >= vq->vq_nentries) { 336 vq->vq_avail_idx -= vq->vq_nentries; 337 vq->vq_packed.cached_flags ^= 338 VRING_PACKED_DESC_F_AVAIL_USED; 339 flags = vq->vq_packed.cached_flags; 340 } 341 } 342 343 static inline int 344 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq, 345 struct rte_mbuf **cookie, uint16_t num) 346 { 347 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 348 struct vq_desc_extra *dxp; 349 uint16_t idx; 350 int i; 351 352 if (unlikely(vq->vq_free_cnt == 0)) 353 return -ENOSPC; 354 if (unlikely(vq->vq_free_cnt < num)) 355 return -EMSGSIZE; 356 357 for (i = 0; i < num; i++) { 358 idx = vq->vq_avail_idx; 359 dxp = &vq->vq_descx[idx]; 360 dxp->cookie = (void *)cookie[i]; 361 dxp->ndescs = 1; 362 363 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 364 } 365 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 366 return 0; 367 } 368 369 static inline int 370 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 371 struct rte_mbuf **cookie, uint16_t num) 372 { 373 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 374 struct vq_desc_extra *dxp; 375 uint16_t idx, did; 376 int i; 377 378 if (unlikely(vq->vq_free_cnt == 0)) 379 return -ENOSPC; 380 if (unlikely(vq->vq_free_cnt < num)) 381 return -EMSGSIZE; 382 383 for (i = 0; i < num; i++) { 384 idx = vq->vq_avail_idx; 385 did = start_dp[idx].id; 386 dxp = &vq->vq_descx[did]; 387 dxp->cookie = (void *)cookie[i]; 388 dxp->ndescs = 1; 389 390 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 391 } 392 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 393 return 0; 394 } 395 396 /* When doing TSO, the IP length is not included in the pseudo header 397 * checksum of the packet given to the PMD, but for virtio it is 398 * expected. 399 */ 400 static void 401 virtio_tso_fix_cksum(struct rte_mbuf *m) 402 { 403 /* common case: header is not fragmented */ 404 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 405 m->l4_len)) { 406 struct rte_ipv4_hdr *iph; 407 struct rte_ipv6_hdr *ip6h; 408 struct rte_tcp_hdr *th; 409 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; 410 uint32_t tmp; 411 412 iph = rte_pktmbuf_mtod_offset(m, 413 struct rte_ipv4_hdr *, m->l2_len); 414 th = RTE_PTR_ADD(iph, m->l3_len); 415 if ((iph->version_ihl >> 4) == 4) { 416 iph->hdr_checksum = 0; 417 iph->hdr_checksum = rte_ipv4_cksum(iph); 418 ip_len = iph->total_length; 419 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - 420 m->l3_len); 421 } else { 422 ip6h = (struct rte_ipv6_hdr *)iph; 423 ip_paylen = ip6h->payload_len; 424 } 425 426 /* calculate the new phdr checksum not including ip_paylen */ 427 prev_cksum = th->cksum; 428 tmp = prev_cksum; 429 tmp += ip_paylen; 430 tmp = (tmp & 0xffff) + (tmp >> 16); 431 new_cksum = tmp; 432 433 /* replace it in the packet */ 434 th->cksum = new_cksum; 435 } 436 } 437 438 439 440 441 static inline void 442 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 443 struct rte_mbuf **cookies, 444 uint16_t num) 445 { 446 struct vq_desc_extra *dxp; 447 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 448 struct vring_desc *start_dp; 449 struct virtio_net_hdr *hdr; 450 uint16_t idx; 451 int16_t head_size = vq->hw->vtnet_hdr_size; 452 uint16_t i = 0; 453 454 idx = vq->vq_desc_head_idx; 455 start_dp = vq->vq_split.ring.desc; 456 457 while (i < num) { 458 idx = idx & (vq->vq_nentries - 1); 459 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 460 dxp->cookie = (void *)cookies[i]; 461 dxp->ndescs = 1; 462 virtio_update_packet_stats(&txvq->stats, cookies[i]); 463 464 hdr = rte_pktmbuf_mtod_offset(cookies[i], 465 struct virtio_net_hdr *, -head_size); 466 467 /* if offload disabled, hdr is not zeroed yet, do it now */ 468 if (!vq->hw->has_tx_offload) 469 virtqueue_clear_net_hdr(hdr); 470 else 471 virtqueue_xmit_offload(hdr, cookies[i]); 472 473 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size; 474 start_dp[idx].len = cookies[i]->data_len + head_size; 475 start_dp[idx].flags = 0; 476 477 478 vq_update_avail_ring(vq, idx); 479 480 idx++; 481 i++; 482 }; 483 484 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 485 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 486 } 487 488 static inline void 489 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 490 struct rte_mbuf *cookie, 491 int in_order) 492 { 493 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 494 struct vring_packed_desc *dp; 495 struct vq_desc_extra *dxp; 496 uint16_t idx, id, flags; 497 int16_t head_size = vq->hw->vtnet_hdr_size; 498 struct virtio_net_hdr *hdr; 499 500 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 501 idx = vq->vq_avail_idx; 502 dp = &vq->vq_packed.ring.desc[idx]; 503 504 dxp = &vq->vq_descx[id]; 505 dxp->ndescs = 1; 506 dxp->cookie = cookie; 507 508 flags = vq->vq_packed.cached_flags; 509 510 /* prepend cannot fail, checked by caller */ 511 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 512 -head_size); 513 514 /* if offload disabled, hdr is not zeroed yet, do it now */ 515 if (!vq->hw->has_tx_offload) 516 virtqueue_clear_net_hdr(hdr); 517 else 518 virtqueue_xmit_offload(hdr, cookie); 519 520 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; 521 dp->len = cookie->data_len + head_size; 522 dp->id = id; 523 524 if (++vq->vq_avail_idx >= vq->vq_nentries) { 525 vq->vq_avail_idx -= vq->vq_nentries; 526 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 527 } 528 529 vq->vq_free_cnt--; 530 531 if (!in_order) { 532 vq->vq_desc_head_idx = dxp->next; 533 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 534 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 535 } 536 537 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 538 } 539 540 static inline void 541 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 542 uint16_t needed, int use_indirect, int can_push, 543 int in_order) 544 { 545 struct virtio_tx_region *txr = txvq->hdr_mz->addr; 546 struct vq_desc_extra *dxp; 547 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 548 struct vring_desc *start_dp; 549 uint16_t seg_num = cookie->nb_segs; 550 uint16_t head_idx, idx; 551 int16_t head_size = vq->hw->vtnet_hdr_size; 552 bool prepend_header = false; 553 struct virtio_net_hdr *hdr; 554 555 head_idx = vq->vq_desc_head_idx; 556 idx = head_idx; 557 if (in_order) 558 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 559 else 560 dxp = &vq->vq_descx[idx]; 561 dxp->cookie = (void *)cookie; 562 dxp->ndescs = needed; 563 564 start_dp = vq->vq_split.ring.desc; 565 566 if (can_push) { 567 /* prepend cannot fail, checked by caller */ 568 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 569 -head_size); 570 prepend_header = true; 571 572 /* if offload disabled, it is not zeroed below, do it now */ 573 if (!vq->hw->has_tx_offload) 574 virtqueue_clear_net_hdr(hdr); 575 } else if (use_indirect) { 576 /* setup tx ring slot to point to indirect 577 * descriptor list stored in reserved region. 578 * 579 * the first slot in indirect ring is already preset 580 * to point to the header in reserved region 581 */ 582 start_dp[idx].addr = txvq->hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 583 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 584 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 585 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 586 587 /* loop below will fill in rest of the indirect elements */ 588 start_dp = txr[idx].tx_indir; 589 idx = 1; 590 } else { 591 /* setup first tx ring slot to point to header 592 * stored in reserved region. 593 */ 594 start_dp[idx].addr = txvq->hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 595 start_dp[idx].len = vq->hw->vtnet_hdr_size; 596 start_dp[idx].flags = VRING_DESC_F_NEXT; 597 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 598 599 idx = start_dp[idx].next; 600 } 601 602 if (vq->hw->has_tx_offload) 603 virtqueue_xmit_offload(hdr, cookie); 604 605 do { 606 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 607 start_dp[idx].len = cookie->data_len; 608 if (prepend_header) { 609 start_dp[idx].addr -= head_size; 610 start_dp[idx].len += head_size; 611 prepend_header = false; 612 } 613 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 614 idx = start_dp[idx].next; 615 } while ((cookie = cookie->next) != NULL); 616 617 if (use_indirect) 618 idx = vq->vq_split.ring.desc[head_idx].next; 619 620 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 621 622 vq->vq_desc_head_idx = idx; 623 vq_update_avail_ring(vq, head_idx); 624 625 if (!in_order) { 626 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 627 vq->vq_desc_tail_idx = idx; 628 } 629 } 630 631 void 632 virtio_dev_cq_start(struct rte_eth_dev *dev) 633 { 634 struct virtio_hw *hw = dev->data->dev_private; 635 636 if (hw->cvq) { 637 rte_spinlock_init(&hw->cvq->lock); 638 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); 639 } 640 } 641 642 int 643 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 644 uint16_t queue_idx, 645 uint16_t nb_desc, 646 unsigned int socket_id __rte_unused, 647 const struct rte_eth_rxconf *rx_conf, 648 struct rte_mempool *mp) 649 { 650 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 651 struct virtio_hw *hw = dev->data->dev_private; 652 struct virtqueue *vq = hw->vqs[vq_idx]; 653 struct virtnet_rx *rxvq; 654 uint16_t rx_free_thresh; 655 uint16_t buf_size; 656 const char *error; 657 658 PMD_INIT_FUNC_TRACE(); 659 660 if (rx_conf->rx_deferred_start) { 661 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 662 return -EINVAL; 663 } 664 665 buf_size = virtio_rx_mem_pool_buf_size(mp); 666 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size, 667 hw->rx_ol_scatter, &error)) { 668 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s", 669 queue_idx, error); 670 return -EINVAL; 671 } 672 673 rx_free_thresh = rx_conf->rx_free_thresh; 674 if (rx_free_thresh == 0) 675 rx_free_thresh = 676 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); 677 678 if (rx_free_thresh & 0x3) { 679 PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four." 680 " (rx_free_thresh=%u port=%u queue=%u)", 681 rx_free_thresh, dev->data->port_id, queue_idx); 682 return -EINVAL; 683 } 684 685 if (rx_free_thresh >= vq->vq_nentries) { 686 PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the " 687 "number of RX entries (%u)." 688 " (rx_free_thresh=%u port=%u queue=%u)", 689 vq->vq_nentries, 690 rx_free_thresh, dev->data->port_id, queue_idx); 691 return -EINVAL; 692 } 693 vq->vq_free_thresh = rx_free_thresh; 694 695 /* 696 * For split ring vectorized path descriptors number must be 697 * equal to the ring size. 698 */ 699 if (nb_desc > vq->vq_nentries || 700 (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) { 701 nb_desc = vq->vq_nentries; 702 } 703 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 704 705 rxvq = &vq->rxq; 706 rxvq->queue_id = queue_idx; 707 rxvq->mpool = mp; 708 dev->data->rx_queues[queue_idx] = rxvq; 709 710 return 0; 711 } 712 713 int 714 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 715 { 716 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 717 struct virtio_hw *hw = dev->data->dev_private; 718 struct virtqueue *vq = hw->vqs[vq_idx]; 719 struct virtnet_rx *rxvq = &vq->rxq; 720 struct rte_mbuf *m; 721 uint16_t desc_idx; 722 int error, nbufs, i; 723 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 724 725 PMD_INIT_FUNC_TRACE(); 726 727 /* Allocate blank mbufs for the each rx descriptor */ 728 nbufs = 0; 729 730 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 731 for (desc_idx = 0; desc_idx < vq->vq_nentries; 732 desc_idx++) { 733 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 734 vq->vq_split.ring.desc[desc_idx].flags = 735 VRING_DESC_F_WRITE; 736 } 737 738 virtio_rxq_vec_setup(rxvq); 739 } 740 741 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); 742 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) 743 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; 744 745 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 746 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 747 virtio_rxq_rearm_vec(rxvq); 748 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 749 } 750 } else if (!virtio_with_packed_queue(vq->hw) && in_order) { 751 if ((!virtqueue_full(vq))) { 752 uint16_t free_cnt = vq->vq_free_cnt; 753 struct rte_mbuf *pkts[free_cnt]; 754 755 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 756 free_cnt)) { 757 error = virtqueue_enqueue_refill_inorder(vq, 758 pkts, 759 free_cnt); 760 if (unlikely(error)) { 761 for (i = 0; i < free_cnt; i++) 762 rte_pktmbuf_free(pkts[i]); 763 } else { 764 nbufs += free_cnt; 765 } 766 } 767 768 vq_update_avail_idx(vq); 769 } 770 } else { 771 while (!virtqueue_full(vq)) { 772 m = rte_mbuf_raw_alloc(rxvq->mpool); 773 if (m == NULL) 774 break; 775 776 /* Enqueue allocated buffers */ 777 if (virtio_with_packed_queue(vq->hw)) 778 error = virtqueue_enqueue_recv_refill_packed_init(vq, 779 &m, 1); 780 else 781 error = virtqueue_enqueue_recv_refill(vq, 782 &m, 1); 783 if (error) { 784 rte_pktmbuf_free(m); 785 break; 786 } 787 nbufs++; 788 } 789 790 if (!virtio_with_packed_queue(vq->hw)) 791 vq_update_avail_idx(vq); 792 } 793 794 PMD_INIT_LOG(DEBUG, "Allocated %d bufs (port=%u queue=%u)", nbufs, 795 dev->data->port_id, queue_idx); 796 797 VIRTQUEUE_DUMP(vq); 798 799 return 0; 800 } 801 802 /* 803 * struct rte_eth_dev *dev: Used to update dev 804 * uint16_t nb_desc: Defaults to values read from config space 805 * unsigned int socket_id: Used to allocate memzone 806 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 807 * uint16_t queue_idx: Just used as an index in dev txq list 808 */ 809 int 810 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 811 uint16_t queue_idx, 812 uint16_t nb_desc, 813 unsigned int socket_id __rte_unused, 814 const struct rte_eth_txconf *tx_conf) 815 { 816 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 817 struct virtio_hw *hw = dev->data->dev_private; 818 struct virtqueue *vq = hw->vqs[vq_idx]; 819 struct virtnet_tx *txvq; 820 uint16_t tx_free_thresh; 821 822 PMD_INIT_FUNC_TRACE(); 823 824 if (tx_conf->tx_deferred_start) { 825 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 826 return -EINVAL; 827 } 828 829 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 830 nb_desc = vq->vq_nentries; 831 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 832 833 txvq = &vq->txq; 834 txvq->queue_id = queue_idx; 835 836 tx_free_thresh = tx_conf->tx_free_thresh; 837 if (tx_free_thresh == 0) 838 tx_free_thresh = 839 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 840 841 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 842 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " 843 "number of TX entries minus 3 (%u)." 844 " (tx_free_thresh=%u port=%u queue=%u)", 845 vq->vq_nentries - 3, 846 tx_free_thresh, dev->data->port_id, queue_idx); 847 return -EINVAL; 848 } 849 850 vq->vq_free_thresh = tx_free_thresh; 851 852 dev->data->tx_queues[queue_idx] = txvq; 853 return 0; 854 } 855 856 int 857 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 858 uint16_t queue_idx) 859 { 860 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 861 struct virtio_hw *hw = dev->data->dev_private; 862 struct virtqueue *vq = hw->vqs[vq_idx]; 863 864 PMD_INIT_FUNC_TRACE(); 865 866 if (!virtio_with_packed_queue(hw)) { 867 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 868 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 869 } 870 871 VIRTQUEUE_DUMP(vq); 872 873 return 0; 874 } 875 876 static inline void 877 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 878 { 879 int error; 880 /* 881 * Requeue the discarded mbuf. This should always be 882 * successful since it was just dequeued. 883 */ 884 if (virtio_with_packed_queue(vq->hw)) 885 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 886 else 887 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 888 889 if (unlikely(error)) { 890 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 891 rte_pktmbuf_free(m); 892 } 893 } 894 895 static inline void 896 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 897 { 898 int error; 899 900 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 901 if (unlikely(error)) { 902 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 903 rte_pktmbuf_free(m); 904 } 905 } 906 907 /* Optionally fill offload information in structure */ 908 static inline int 909 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 910 { 911 struct rte_net_hdr_lens hdr_lens; 912 uint32_t hdrlen, ptype; 913 int l4_supported = 0; 914 915 /* nothing to do */ 916 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 917 return 0; 918 919 m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN; 920 921 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 922 m->packet_type = ptype; 923 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 924 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 925 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 926 l4_supported = 1; 927 928 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 929 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 930 if (hdr->csum_start <= hdrlen && l4_supported) { 931 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_NONE; 932 } else { 933 /* Unknown proto or tunnel, do sw cksum. We can assume 934 * the cksum field is in the first segment since the 935 * buffers we provided to the host are large enough. 936 * In case of SCTP, this will be wrong since it's a CRC 937 * but there's nothing we can do. 938 */ 939 uint16_t csum = 0, off; 940 941 if (rte_raw_cksum_mbuf(m, hdr->csum_start, 942 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 943 &csum) < 0) 944 return -EINVAL; 945 if (likely(csum != 0xffff)) 946 csum = ~csum; 947 off = hdr->csum_offset + hdr->csum_start; 948 if (rte_pktmbuf_data_len(m) >= off + 1) 949 *rte_pktmbuf_mtod_offset(m, uint16_t *, 950 off) = csum; 951 } 952 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 953 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 954 } 955 956 /* GSO request, save required information in mbuf */ 957 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 958 /* Check unsupported modes */ 959 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 960 (hdr->gso_size == 0)) { 961 return -EINVAL; 962 } 963 964 /* Update mss lengths in mbuf */ 965 m->tso_segsz = hdr->gso_size; 966 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 967 case VIRTIO_NET_HDR_GSO_TCPV4: 968 case VIRTIO_NET_HDR_GSO_TCPV6: 969 m->ol_flags |= RTE_MBUF_F_RX_LRO | 970 RTE_MBUF_F_RX_L4_CKSUM_NONE; 971 break; 972 default: 973 return -EINVAL; 974 } 975 } 976 977 return 0; 978 } 979 980 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 981 uint16_t 982 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 983 { 984 struct virtnet_rx *rxvq = rx_queue; 985 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 986 struct virtio_hw *hw = vq->hw; 987 struct rte_mbuf *rxm; 988 uint16_t nb_used, num, nb_rx; 989 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 990 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 991 int error; 992 uint32_t i, nb_enqueued; 993 uint32_t hdr_size; 994 struct virtio_net_hdr *hdr; 995 996 nb_rx = 0; 997 if (unlikely(hw->started == 0)) 998 return nb_rx; 999 1000 nb_used = virtqueue_nused(vq); 1001 1002 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1003 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1004 num = VIRTIO_MBUF_BURST_SZ; 1005 if (likely(num > DESC_PER_CACHELINE)) 1006 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1007 1008 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1009 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 1010 1011 nb_enqueued = 0; 1012 hdr_size = hw->vtnet_hdr_size; 1013 1014 for (i = 0; i < num ; i++) { 1015 rxm = rcv_pkts[i]; 1016 1017 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1018 1019 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1020 PMD_RX_LOG(ERR, "Packet drop"); 1021 nb_enqueued++; 1022 virtio_discard_rxbuf(vq, rxm); 1023 rxvq->stats.errors++; 1024 continue; 1025 } 1026 1027 rxm->port = rxvq->port_id; 1028 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1029 rxm->ol_flags = 0; 1030 rxm->vlan_tci = 0; 1031 1032 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1033 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1034 1035 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1036 RTE_PKTMBUF_HEADROOM - hdr_size); 1037 1038 if (hw->vlan_strip) 1039 rte_vlan_strip(rxm); 1040 1041 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1042 virtio_discard_rxbuf(vq, rxm); 1043 rxvq->stats.errors++; 1044 continue; 1045 } 1046 1047 virtio_rx_stats_updated(rxvq, rxm); 1048 1049 rx_pkts[nb_rx++] = rxm; 1050 } 1051 1052 rxvq->stats.packets += nb_rx; 1053 1054 /* Allocate new mbuf for the used descriptor */ 1055 if (likely(!virtqueue_full(vq))) { 1056 uint16_t free_cnt = vq->vq_free_cnt; 1057 struct rte_mbuf *new_pkts[free_cnt]; 1058 1059 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1060 free_cnt) == 0)) { 1061 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1062 free_cnt); 1063 if (unlikely(error)) { 1064 for (i = 0; i < free_cnt; i++) 1065 rte_pktmbuf_free(new_pkts[i]); 1066 } 1067 nb_enqueued += free_cnt; 1068 } else { 1069 struct rte_eth_dev *dev = 1070 &rte_eth_devices[rxvq->port_id]; 1071 dev->data->rx_mbuf_alloc_failed += free_cnt; 1072 } 1073 } 1074 1075 if (likely(nb_enqueued)) { 1076 vq_update_avail_idx(vq); 1077 1078 if (unlikely(virtqueue_kick_prepare(vq))) { 1079 virtqueue_notify(vq); 1080 PMD_RX_LOG(DEBUG, "Notified"); 1081 } 1082 } 1083 1084 return nb_rx; 1085 } 1086 1087 uint16_t 1088 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1089 uint16_t nb_pkts) 1090 { 1091 struct virtnet_rx *rxvq = rx_queue; 1092 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1093 struct virtio_hw *hw = vq->hw; 1094 struct rte_mbuf *rxm; 1095 uint16_t num, nb_rx; 1096 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1097 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1098 int error; 1099 uint32_t i, nb_enqueued; 1100 uint32_t hdr_size; 1101 struct virtio_net_hdr *hdr; 1102 1103 nb_rx = 0; 1104 if (unlikely(hw->started == 0)) 1105 return nb_rx; 1106 1107 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1108 if (likely(num > DESC_PER_CACHELINE)) 1109 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1110 1111 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1112 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1113 1114 nb_enqueued = 0; 1115 hdr_size = hw->vtnet_hdr_size; 1116 1117 for (i = 0; i < num; i++) { 1118 rxm = rcv_pkts[i]; 1119 1120 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1121 1122 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1123 PMD_RX_LOG(ERR, "Packet drop"); 1124 nb_enqueued++; 1125 virtio_discard_rxbuf(vq, rxm); 1126 rxvq->stats.errors++; 1127 continue; 1128 } 1129 1130 rxm->port = rxvq->port_id; 1131 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1132 rxm->ol_flags = 0; 1133 rxm->vlan_tci = 0; 1134 1135 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1136 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1137 1138 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1139 RTE_PKTMBUF_HEADROOM - hdr_size); 1140 1141 if (hw->vlan_strip) 1142 rte_vlan_strip(rxm); 1143 1144 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1145 virtio_discard_rxbuf(vq, rxm); 1146 rxvq->stats.errors++; 1147 continue; 1148 } 1149 1150 virtio_rx_stats_updated(rxvq, rxm); 1151 1152 rx_pkts[nb_rx++] = rxm; 1153 } 1154 1155 rxvq->stats.packets += nb_rx; 1156 1157 /* Allocate new mbuf for the used descriptor */ 1158 if (likely(!virtqueue_full(vq))) { 1159 uint16_t free_cnt = vq->vq_free_cnt; 1160 struct rte_mbuf *new_pkts[free_cnt]; 1161 1162 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1163 free_cnt) == 0)) { 1164 error = virtqueue_enqueue_recv_refill_packed(vq, 1165 new_pkts, free_cnt); 1166 if (unlikely(error)) { 1167 for (i = 0; i < free_cnt; i++) 1168 rte_pktmbuf_free(new_pkts[i]); 1169 } 1170 nb_enqueued += free_cnt; 1171 } else { 1172 struct rte_eth_dev *dev = 1173 &rte_eth_devices[rxvq->port_id]; 1174 dev->data->rx_mbuf_alloc_failed += free_cnt; 1175 } 1176 } 1177 1178 if (likely(nb_enqueued)) { 1179 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1180 virtqueue_notify(vq); 1181 PMD_RX_LOG(DEBUG, "Notified"); 1182 } 1183 } 1184 1185 return nb_rx; 1186 } 1187 1188 1189 uint16_t 1190 virtio_recv_pkts_inorder(void *rx_queue, 1191 struct rte_mbuf **rx_pkts, 1192 uint16_t nb_pkts) 1193 { 1194 struct virtnet_rx *rxvq = rx_queue; 1195 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1196 struct virtio_hw *hw = vq->hw; 1197 struct rte_mbuf *rxm; 1198 struct rte_mbuf *prev = NULL; 1199 uint16_t nb_used, num, nb_rx; 1200 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1201 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1202 int error; 1203 uint32_t nb_enqueued; 1204 uint32_t seg_num; 1205 uint32_t seg_res; 1206 uint32_t hdr_size; 1207 int32_t i; 1208 1209 nb_rx = 0; 1210 if (unlikely(hw->started == 0)) 1211 return nb_rx; 1212 1213 nb_used = virtqueue_nused(vq); 1214 nb_used = RTE_MIN(nb_used, nb_pkts); 1215 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1216 1217 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1218 1219 nb_enqueued = 0; 1220 seg_num = 1; 1221 seg_res = 0; 1222 hdr_size = hw->vtnet_hdr_size; 1223 1224 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1225 1226 for (i = 0; i < num; i++) { 1227 struct virtio_net_hdr_mrg_rxbuf *header; 1228 1229 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1230 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1231 1232 rxm = rcv_pkts[i]; 1233 1234 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1235 PMD_RX_LOG(ERR, "Packet drop"); 1236 nb_enqueued++; 1237 virtio_discard_rxbuf_inorder(vq, rxm); 1238 rxvq->stats.errors++; 1239 continue; 1240 } 1241 1242 header = (struct virtio_net_hdr_mrg_rxbuf *) 1243 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1244 - hdr_size); 1245 1246 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1247 seg_num = header->num_buffers; 1248 if (seg_num == 0) 1249 seg_num = 1; 1250 } else { 1251 seg_num = 1; 1252 } 1253 1254 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1255 rxm->nb_segs = seg_num; 1256 rxm->ol_flags = 0; 1257 rxm->vlan_tci = 0; 1258 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1259 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1260 1261 rxm->port = rxvq->port_id; 1262 1263 rx_pkts[nb_rx] = rxm; 1264 prev = rxm; 1265 1266 if (vq->hw->has_rx_offload && 1267 virtio_rx_offload(rxm, &header->hdr) < 0) { 1268 virtio_discard_rxbuf_inorder(vq, rxm); 1269 rxvq->stats.errors++; 1270 continue; 1271 } 1272 1273 if (hw->vlan_strip) 1274 rte_vlan_strip(rx_pkts[nb_rx]); 1275 1276 seg_res = seg_num - 1; 1277 1278 /* Merge remaining segments */ 1279 while (seg_res != 0 && i < (num - 1)) { 1280 i++; 1281 1282 rxm = rcv_pkts[i]; 1283 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1284 rxm->pkt_len = (uint32_t)(len[i]); 1285 rxm->data_len = (uint16_t)(len[i]); 1286 1287 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1288 1289 prev->next = rxm; 1290 prev = rxm; 1291 seg_res -= 1; 1292 } 1293 1294 if (!seg_res) { 1295 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1296 nb_rx++; 1297 } 1298 } 1299 1300 /* Last packet still need merge segments */ 1301 while (seg_res != 0) { 1302 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1303 VIRTIO_MBUF_BURST_SZ); 1304 1305 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1306 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1307 rcv_cnt); 1308 uint16_t extra_idx = 0; 1309 1310 rcv_cnt = num; 1311 while (extra_idx < rcv_cnt) { 1312 rxm = rcv_pkts[extra_idx]; 1313 rxm->data_off = 1314 RTE_PKTMBUF_HEADROOM - hdr_size; 1315 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1316 rxm->data_len = (uint16_t)(len[extra_idx]); 1317 prev->next = rxm; 1318 prev = rxm; 1319 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1320 extra_idx += 1; 1321 }; 1322 seg_res -= rcv_cnt; 1323 1324 if (!seg_res) { 1325 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1326 nb_rx++; 1327 } 1328 } else { 1329 PMD_RX_LOG(ERR, 1330 "No enough segments for packet."); 1331 rte_pktmbuf_free(rx_pkts[nb_rx]); 1332 rxvq->stats.errors++; 1333 break; 1334 } 1335 } 1336 1337 rxvq->stats.packets += nb_rx; 1338 1339 /* Allocate new mbuf for the used descriptor */ 1340 1341 if (likely(!virtqueue_full(vq))) { 1342 /* free_cnt may include mrg descs */ 1343 uint16_t free_cnt = vq->vq_free_cnt; 1344 struct rte_mbuf *new_pkts[free_cnt]; 1345 1346 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1347 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1348 free_cnt); 1349 if (unlikely(error)) { 1350 for (i = 0; i < free_cnt; i++) 1351 rte_pktmbuf_free(new_pkts[i]); 1352 } 1353 nb_enqueued += free_cnt; 1354 } else { 1355 struct rte_eth_dev *dev = 1356 &rte_eth_devices[rxvq->port_id]; 1357 dev->data->rx_mbuf_alloc_failed += free_cnt; 1358 } 1359 } 1360 1361 if (likely(nb_enqueued)) { 1362 vq_update_avail_idx(vq); 1363 1364 if (unlikely(virtqueue_kick_prepare(vq))) { 1365 virtqueue_notify(vq); 1366 PMD_RX_LOG(DEBUG, "Notified"); 1367 } 1368 } 1369 1370 return nb_rx; 1371 } 1372 1373 uint16_t 1374 virtio_recv_mergeable_pkts(void *rx_queue, 1375 struct rte_mbuf **rx_pkts, 1376 uint16_t nb_pkts) 1377 { 1378 struct virtnet_rx *rxvq = rx_queue; 1379 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1380 struct virtio_hw *hw = vq->hw; 1381 struct rte_mbuf *rxm; 1382 struct rte_mbuf *prev = NULL; 1383 uint16_t nb_used, num, nb_rx = 0; 1384 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1385 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1386 int error; 1387 uint32_t nb_enqueued = 0; 1388 uint32_t seg_num = 0; 1389 uint32_t seg_res = 0; 1390 uint32_t hdr_size = hw->vtnet_hdr_size; 1391 int32_t i; 1392 1393 if (unlikely(hw->started == 0)) 1394 return nb_rx; 1395 1396 nb_used = virtqueue_nused(vq); 1397 1398 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1399 1400 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1401 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1402 num = VIRTIO_MBUF_BURST_SZ; 1403 if (likely(num > DESC_PER_CACHELINE)) 1404 num = num - ((vq->vq_used_cons_idx + num) % 1405 DESC_PER_CACHELINE); 1406 1407 1408 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1409 1410 for (i = 0; i < num; i++) { 1411 struct virtio_net_hdr_mrg_rxbuf *header; 1412 1413 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1414 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1415 1416 rxm = rcv_pkts[i]; 1417 1418 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1419 PMD_RX_LOG(ERR, "Packet drop"); 1420 nb_enqueued++; 1421 virtio_discard_rxbuf(vq, rxm); 1422 rxvq->stats.errors++; 1423 continue; 1424 } 1425 1426 header = (struct virtio_net_hdr_mrg_rxbuf *) 1427 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1428 - hdr_size); 1429 seg_num = header->num_buffers; 1430 if (seg_num == 0) 1431 seg_num = 1; 1432 1433 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1434 rxm->nb_segs = seg_num; 1435 rxm->ol_flags = 0; 1436 rxm->vlan_tci = 0; 1437 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1438 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1439 1440 rxm->port = rxvq->port_id; 1441 1442 rx_pkts[nb_rx] = rxm; 1443 prev = rxm; 1444 1445 if (hw->has_rx_offload && 1446 virtio_rx_offload(rxm, &header->hdr) < 0) { 1447 virtio_discard_rxbuf(vq, rxm); 1448 rxvq->stats.errors++; 1449 continue; 1450 } 1451 1452 if (hw->vlan_strip) 1453 rte_vlan_strip(rx_pkts[nb_rx]); 1454 1455 seg_res = seg_num - 1; 1456 1457 /* Merge remaining segments */ 1458 while (seg_res != 0 && i < (num - 1)) { 1459 i++; 1460 1461 rxm = rcv_pkts[i]; 1462 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1463 rxm->pkt_len = (uint32_t)(len[i]); 1464 rxm->data_len = (uint16_t)(len[i]); 1465 1466 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1467 1468 prev->next = rxm; 1469 prev = rxm; 1470 seg_res -= 1; 1471 } 1472 1473 if (!seg_res) { 1474 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1475 nb_rx++; 1476 } 1477 } 1478 1479 /* Last packet still need merge segments */ 1480 while (seg_res != 0) { 1481 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1482 VIRTIO_MBUF_BURST_SZ); 1483 1484 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1485 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1486 rcv_cnt); 1487 uint16_t extra_idx = 0; 1488 1489 rcv_cnt = num; 1490 while (extra_idx < rcv_cnt) { 1491 rxm = rcv_pkts[extra_idx]; 1492 rxm->data_off = 1493 RTE_PKTMBUF_HEADROOM - hdr_size; 1494 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1495 rxm->data_len = (uint16_t)(len[extra_idx]); 1496 prev->next = rxm; 1497 prev = rxm; 1498 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1499 extra_idx += 1; 1500 }; 1501 seg_res -= rcv_cnt; 1502 1503 if (!seg_res) { 1504 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1505 nb_rx++; 1506 } 1507 } else { 1508 PMD_RX_LOG(ERR, 1509 "No enough segments for packet."); 1510 rte_pktmbuf_free(rx_pkts[nb_rx]); 1511 rxvq->stats.errors++; 1512 break; 1513 } 1514 } 1515 1516 rxvq->stats.packets += nb_rx; 1517 1518 /* Allocate new mbuf for the used descriptor */ 1519 if (likely(!virtqueue_full(vq))) { 1520 /* free_cnt may include mrg descs */ 1521 uint16_t free_cnt = vq->vq_free_cnt; 1522 struct rte_mbuf *new_pkts[free_cnt]; 1523 1524 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1525 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1526 free_cnt); 1527 if (unlikely(error)) { 1528 for (i = 0; i < free_cnt; i++) 1529 rte_pktmbuf_free(new_pkts[i]); 1530 } 1531 nb_enqueued += free_cnt; 1532 } else { 1533 struct rte_eth_dev *dev = 1534 &rte_eth_devices[rxvq->port_id]; 1535 dev->data->rx_mbuf_alloc_failed += free_cnt; 1536 } 1537 } 1538 1539 if (likely(nb_enqueued)) { 1540 vq_update_avail_idx(vq); 1541 1542 if (unlikely(virtqueue_kick_prepare(vq))) { 1543 virtqueue_notify(vq); 1544 PMD_RX_LOG(DEBUG, "Notified"); 1545 } 1546 } 1547 1548 return nb_rx; 1549 } 1550 1551 uint16_t 1552 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1553 struct rte_mbuf **rx_pkts, 1554 uint16_t nb_pkts) 1555 { 1556 struct virtnet_rx *rxvq = rx_queue; 1557 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1558 struct virtio_hw *hw = vq->hw; 1559 struct rte_mbuf *rxm; 1560 struct rte_mbuf *prev = NULL; 1561 uint16_t num, nb_rx = 0; 1562 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1563 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1564 uint32_t nb_enqueued = 0; 1565 uint32_t seg_num = 0; 1566 uint32_t seg_res = 0; 1567 uint32_t hdr_size = hw->vtnet_hdr_size; 1568 int32_t i; 1569 int error; 1570 1571 if (unlikely(hw->started == 0)) 1572 return nb_rx; 1573 1574 1575 num = nb_pkts; 1576 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1577 num = VIRTIO_MBUF_BURST_SZ; 1578 if (likely(num > DESC_PER_CACHELINE)) 1579 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1580 1581 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1582 1583 for (i = 0; i < num; i++) { 1584 struct virtio_net_hdr_mrg_rxbuf *header; 1585 1586 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1587 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1588 1589 rxm = rcv_pkts[i]; 1590 1591 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1592 PMD_RX_LOG(ERR, "Packet drop"); 1593 nb_enqueued++; 1594 virtio_discard_rxbuf(vq, rxm); 1595 rxvq->stats.errors++; 1596 continue; 1597 } 1598 1599 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1600 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1601 seg_num = header->num_buffers; 1602 1603 if (seg_num == 0) 1604 seg_num = 1; 1605 1606 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1607 rxm->nb_segs = seg_num; 1608 rxm->ol_flags = 0; 1609 rxm->vlan_tci = 0; 1610 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1611 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1612 1613 rxm->port = rxvq->port_id; 1614 rx_pkts[nb_rx] = rxm; 1615 prev = rxm; 1616 1617 if (hw->has_rx_offload && 1618 virtio_rx_offload(rxm, &header->hdr) < 0) { 1619 virtio_discard_rxbuf(vq, rxm); 1620 rxvq->stats.errors++; 1621 continue; 1622 } 1623 1624 if (hw->vlan_strip) 1625 rte_vlan_strip(rx_pkts[nb_rx]); 1626 1627 seg_res = seg_num - 1; 1628 1629 /* Merge remaining segments */ 1630 while (seg_res != 0 && i < (num - 1)) { 1631 i++; 1632 1633 rxm = rcv_pkts[i]; 1634 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1635 rxm->pkt_len = (uint32_t)(len[i]); 1636 rxm->data_len = (uint16_t)(len[i]); 1637 1638 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1639 1640 prev->next = rxm; 1641 prev = rxm; 1642 seg_res -= 1; 1643 } 1644 1645 if (!seg_res) { 1646 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1647 nb_rx++; 1648 } 1649 } 1650 1651 /* Last packet still need merge segments */ 1652 while (seg_res != 0) { 1653 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1654 VIRTIO_MBUF_BURST_SZ); 1655 uint16_t extra_idx = 0; 1656 1657 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1658 len, rcv_cnt); 1659 if (unlikely(rcv_cnt == 0)) { 1660 PMD_RX_LOG(ERR, "No enough segments for packet."); 1661 rte_pktmbuf_free(rx_pkts[nb_rx]); 1662 rxvq->stats.errors++; 1663 break; 1664 } 1665 1666 while (extra_idx < rcv_cnt) { 1667 rxm = rcv_pkts[extra_idx]; 1668 1669 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1670 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1671 rxm->data_len = (uint16_t)(len[extra_idx]); 1672 1673 prev->next = rxm; 1674 prev = rxm; 1675 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1676 extra_idx += 1; 1677 } 1678 seg_res -= rcv_cnt; 1679 if (!seg_res) { 1680 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1681 nb_rx++; 1682 } 1683 } 1684 1685 rxvq->stats.packets += nb_rx; 1686 1687 /* Allocate new mbuf for the used descriptor */ 1688 if (likely(!virtqueue_full(vq))) { 1689 /* free_cnt may include mrg descs */ 1690 uint16_t free_cnt = vq->vq_free_cnt; 1691 struct rte_mbuf *new_pkts[free_cnt]; 1692 1693 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1694 error = virtqueue_enqueue_recv_refill_packed(vq, 1695 new_pkts, free_cnt); 1696 if (unlikely(error)) { 1697 for (i = 0; i < free_cnt; i++) 1698 rte_pktmbuf_free(new_pkts[i]); 1699 } 1700 nb_enqueued += free_cnt; 1701 } else { 1702 struct rte_eth_dev *dev = 1703 &rte_eth_devices[rxvq->port_id]; 1704 dev->data->rx_mbuf_alloc_failed += free_cnt; 1705 } 1706 } 1707 1708 if (likely(nb_enqueued)) { 1709 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1710 virtqueue_notify(vq); 1711 PMD_RX_LOG(DEBUG, "Notified"); 1712 } 1713 } 1714 1715 return nb_rx; 1716 } 1717 1718 uint16_t 1719 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1720 uint16_t nb_pkts) 1721 { 1722 uint16_t nb_tx; 1723 int error; 1724 1725 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1726 struct rte_mbuf *m = tx_pkts[nb_tx]; 1727 1728 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1729 error = rte_validate_tx_offload(m); 1730 if (unlikely(error)) { 1731 rte_errno = -error; 1732 break; 1733 } 1734 #endif 1735 1736 /* Do VLAN tag insertion */ 1737 if (unlikely(m->ol_flags & RTE_MBUF_F_TX_VLAN)) { 1738 error = rte_vlan_insert(&m); 1739 /* rte_vlan_insert() may change pointer 1740 * even in the case of failure 1741 */ 1742 tx_pkts[nb_tx] = m; 1743 1744 if (unlikely(error)) { 1745 rte_errno = -error; 1746 break; 1747 } 1748 } 1749 1750 error = rte_net_intel_cksum_prepare(m); 1751 if (unlikely(error)) { 1752 rte_errno = -error; 1753 break; 1754 } 1755 1756 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1757 virtio_tso_fix_cksum(m); 1758 } 1759 1760 return nb_tx; 1761 } 1762 1763 uint16_t 1764 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 1765 uint16_t nb_pkts) 1766 { 1767 struct virtnet_tx *txvq = tx_queue; 1768 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1769 struct virtio_hw *hw = vq->hw; 1770 uint16_t hdr_size = hw->vtnet_hdr_size; 1771 uint16_t nb_tx = 0; 1772 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 1773 1774 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1775 return nb_tx; 1776 1777 if (unlikely(nb_pkts < 1)) 1778 return nb_pkts; 1779 1780 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1781 1782 if (nb_pkts > vq->vq_free_cnt) 1783 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 1784 in_order); 1785 1786 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1787 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1788 int can_push = 0, use_indirect = 0, slots, need; 1789 1790 /* optimize ring usage */ 1791 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1792 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1793 rte_mbuf_refcnt_read(txm) == 1 && 1794 RTE_MBUF_DIRECT(txm) && 1795 txm->nb_segs == 1 && 1796 rte_pktmbuf_headroom(txm) >= hdr_size && 1797 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1798 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1799 can_push = 1; 1800 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1801 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1802 use_indirect = 1; 1803 /* How many main ring entries are needed to this Tx? 1804 * indirect => 1 1805 * any_layout => number of segments 1806 * default => number of segments + 1 1807 */ 1808 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1809 need = slots - vq->vq_free_cnt; 1810 1811 /* Positive value indicates it need free vring descriptors */ 1812 if (unlikely(need > 0)) { 1813 virtio_xmit_cleanup_packed(vq, need, in_order); 1814 need = slots - vq->vq_free_cnt; 1815 if (unlikely(need > 0)) { 1816 PMD_TX_LOG(ERR, 1817 "No free tx descriptors to transmit"); 1818 break; 1819 } 1820 } 1821 1822 /* Enqueue Packet buffers */ 1823 if (can_push) 1824 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 1825 else 1826 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 1827 use_indirect, 0, 1828 in_order); 1829 1830 virtio_update_packet_stats(&txvq->stats, txm); 1831 } 1832 1833 txvq->stats.packets += nb_tx; 1834 1835 if (likely(nb_tx)) { 1836 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1837 virtqueue_notify(vq); 1838 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1839 } 1840 } 1841 1842 return nb_tx; 1843 } 1844 1845 uint16_t 1846 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1847 { 1848 struct virtnet_tx *txvq = tx_queue; 1849 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1850 struct virtio_hw *hw = vq->hw; 1851 uint16_t hdr_size = hw->vtnet_hdr_size; 1852 uint16_t nb_used, nb_tx = 0; 1853 1854 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1855 return nb_tx; 1856 1857 if (unlikely(nb_pkts < 1)) 1858 return nb_pkts; 1859 1860 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1861 1862 nb_used = virtqueue_nused(vq); 1863 1864 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1865 virtio_xmit_cleanup(vq, nb_used); 1866 1867 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1868 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1869 int can_push = 0, use_indirect = 0, slots, need; 1870 1871 /* optimize ring usage */ 1872 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1873 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1874 rte_mbuf_refcnt_read(txm) == 1 && 1875 RTE_MBUF_DIRECT(txm) && 1876 txm->nb_segs == 1 && 1877 rte_pktmbuf_headroom(txm) >= hdr_size && 1878 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1879 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1880 can_push = 1; 1881 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1882 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1883 use_indirect = 1; 1884 1885 /* How many main ring entries are needed to this Tx? 1886 * any_layout => number of segments 1887 * indirect => 1 1888 * default => number of segments + 1 1889 */ 1890 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1891 need = slots - vq->vq_free_cnt; 1892 1893 /* Positive value indicates it need free vring descriptors */ 1894 if (unlikely(need > 0)) { 1895 nb_used = virtqueue_nused(vq); 1896 1897 need = RTE_MIN(need, (int)nb_used); 1898 1899 virtio_xmit_cleanup(vq, need); 1900 need = slots - vq->vq_free_cnt; 1901 if (unlikely(need > 0)) { 1902 PMD_TX_LOG(ERR, 1903 "No free tx descriptors to transmit"); 1904 break; 1905 } 1906 } 1907 1908 /* Enqueue Packet buffers */ 1909 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 1910 can_push, 0); 1911 1912 virtio_update_packet_stats(&txvq->stats, txm); 1913 } 1914 1915 txvq->stats.packets += nb_tx; 1916 1917 if (likely(nb_tx)) { 1918 vq_update_avail_idx(vq); 1919 1920 if (unlikely(virtqueue_kick_prepare(vq))) { 1921 virtqueue_notify(vq); 1922 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1923 } 1924 } 1925 1926 return nb_tx; 1927 } 1928 1929 static __rte_always_inline int 1930 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 1931 { 1932 uint16_t nb_used, nb_clean, nb_descs; 1933 1934 nb_descs = vq->vq_free_cnt + need; 1935 nb_used = virtqueue_nused(vq); 1936 nb_clean = RTE_MIN(need, (int)nb_used); 1937 1938 virtio_xmit_cleanup_inorder(vq, nb_clean); 1939 1940 return nb_descs - vq->vq_free_cnt; 1941 } 1942 1943 uint16_t 1944 virtio_xmit_pkts_inorder(void *tx_queue, 1945 struct rte_mbuf **tx_pkts, 1946 uint16_t nb_pkts) 1947 { 1948 struct virtnet_tx *txvq = tx_queue; 1949 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1950 struct virtio_hw *hw = vq->hw; 1951 uint16_t hdr_size = hw->vtnet_hdr_size; 1952 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 1953 struct rte_mbuf *inorder_pkts[nb_pkts]; 1954 int need; 1955 1956 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1957 return nb_tx; 1958 1959 if (unlikely(nb_pkts < 1)) 1960 return nb_pkts; 1961 1962 VIRTQUEUE_DUMP(vq); 1963 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1964 nb_used = virtqueue_nused(vq); 1965 1966 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1967 virtio_xmit_cleanup_inorder(vq, nb_used); 1968 1969 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1970 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1971 int slots; 1972 1973 /* optimize ring usage */ 1974 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1975 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1976 rte_mbuf_refcnt_read(txm) == 1 && 1977 RTE_MBUF_DIRECT(txm) && 1978 txm->nb_segs == 1 && 1979 rte_pktmbuf_headroom(txm) >= hdr_size && 1980 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1981 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) { 1982 inorder_pkts[nb_inorder_pkts] = txm; 1983 nb_inorder_pkts++; 1984 1985 continue; 1986 } 1987 1988 if (nb_inorder_pkts) { 1989 need = nb_inorder_pkts - vq->vq_free_cnt; 1990 if (unlikely(need > 0)) { 1991 need = virtio_xmit_try_cleanup_inorder(vq, 1992 need); 1993 if (unlikely(need > 0)) { 1994 PMD_TX_LOG(ERR, 1995 "No free tx descriptors to " 1996 "transmit"); 1997 break; 1998 } 1999 } 2000 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2001 nb_inorder_pkts); 2002 nb_inorder_pkts = 0; 2003 } 2004 2005 slots = txm->nb_segs + 1; 2006 need = slots - vq->vq_free_cnt; 2007 if (unlikely(need > 0)) { 2008 need = virtio_xmit_try_cleanup_inorder(vq, slots); 2009 2010 if (unlikely(need > 0)) { 2011 PMD_TX_LOG(ERR, 2012 "No free tx descriptors to transmit"); 2013 break; 2014 } 2015 } 2016 /* Enqueue Packet buffers */ 2017 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 2018 2019 virtio_update_packet_stats(&txvq->stats, txm); 2020 } 2021 2022 /* Transmit all inorder packets */ 2023 if (nb_inorder_pkts) { 2024 need = nb_inorder_pkts - vq->vq_free_cnt; 2025 if (unlikely(need > 0)) { 2026 need = virtio_xmit_try_cleanup_inorder(vq, 2027 need); 2028 if (unlikely(need > 0)) { 2029 PMD_TX_LOG(ERR, 2030 "No free tx descriptors to transmit"); 2031 nb_inorder_pkts = vq->vq_free_cnt; 2032 nb_tx -= need; 2033 } 2034 } 2035 2036 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2037 nb_inorder_pkts); 2038 } 2039 2040 txvq->stats.packets += nb_tx; 2041 2042 if (likely(nb_tx)) { 2043 vq_update_avail_idx(vq); 2044 2045 if (unlikely(virtqueue_kick_prepare(vq))) { 2046 virtqueue_notify(vq); 2047 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2048 } 2049 } 2050 2051 VIRTQUEUE_DUMP(vq); 2052 2053 return nb_tx; 2054 } 2055 2056 __rte_weak uint16_t 2057 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, 2058 struct rte_mbuf **rx_pkts __rte_unused, 2059 uint16_t nb_pkts __rte_unused) 2060 { 2061 return 0; 2062 } 2063 2064 __rte_weak uint16_t 2065 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, 2066 struct rte_mbuf **tx_pkts __rte_unused, 2067 uint16_t nb_pkts __rte_unused) 2068 { 2069 return 0; 2070 } 2071