1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_cycles.h> 12 #include <rte_memory.h> 13 #include <rte_branch_prediction.h> 14 #include <rte_mempool.h> 15 #include <rte_malloc.h> 16 #include <rte_mbuf.h> 17 #include <rte_ether.h> 18 #include <ethdev_driver.h> 19 #include <rte_prefetch.h> 20 #include <rte_string_fns.h> 21 #include <rte_errno.h> 22 #include <rte_byteorder.h> 23 #include <rte_net.h> 24 #include <rte_ip.h> 25 #include <rte_udp.h> 26 #include <rte_tcp.h> 27 28 #include "virtio_logs.h" 29 #include "virtio_ethdev.h" 30 #include "virtio.h" 31 #include "virtqueue.h" 32 #include "virtio_rxtx.h" 33 #include "virtio_rxtx_simple.h" 34 #include "virtio_ring.h" 35 36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 38 #else 39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 40 #endif 41 42 int 43 virtio_dev_rx_queue_done(void *rxq, uint16_t offset) 44 { 45 struct virtnet_rx *rxvq = rxq; 46 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 47 48 return virtqueue_nused(vq) >= offset; 49 } 50 51 void 52 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 53 { 54 vq->vq_free_cnt += num; 55 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 56 } 57 58 void 59 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 60 { 61 struct vring_desc *dp, *dp_tail; 62 struct vq_desc_extra *dxp; 63 uint16_t desc_idx_last = desc_idx; 64 65 dp = &vq->vq_split.ring.desc[desc_idx]; 66 dxp = &vq->vq_descx[desc_idx]; 67 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 68 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 69 while (dp->flags & VRING_DESC_F_NEXT) { 70 desc_idx_last = dp->next; 71 dp = &vq->vq_split.ring.desc[dp->next]; 72 } 73 } 74 dxp->ndescs = 0; 75 76 /* 77 * We must append the existing free chain, if any, to the end of 78 * newly freed chain. If the virtqueue was completely used, then 79 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 80 */ 81 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 82 vq->vq_desc_head_idx = desc_idx; 83 } else { 84 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 85 dp_tail->next = desc_idx; 86 } 87 88 vq->vq_desc_tail_idx = desc_idx_last; 89 dp->next = VQ_RING_DESC_CHAIN_END; 90 } 91 92 void 93 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf) 94 { 95 uint32_t s = mbuf->pkt_len; 96 struct rte_ether_addr *ea; 97 98 stats->bytes += s; 99 100 if (s == 64) { 101 stats->size_bins[1]++; 102 } else if (s > 64 && s < 1024) { 103 uint32_t bin; 104 105 /* count zeros, and offset into correct bin */ 106 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 107 stats->size_bins[bin]++; 108 } else { 109 if (s < 64) 110 stats->size_bins[0]++; 111 else if (s < 1519) 112 stats->size_bins[6]++; 113 else 114 stats->size_bins[7]++; 115 } 116 117 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 118 if (rte_is_multicast_ether_addr(ea)) { 119 if (rte_is_broadcast_ether_addr(ea)) 120 stats->broadcast++; 121 else 122 stats->multicast++; 123 } 124 } 125 126 static inline void 127 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 128 { 129 VIRTIO_DUMP_PACKET(m, m->data_len); 130 131 virtio_update_packet_stats(&rxvq->stats, m); 132 } 133 134 static uint16_t 135 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 136 struct rte_mbuf **rx_pkts, 137 uint32_t *len, 138 uint16_t num) 139 { 140 struct rte_mbuf *cookie; 141 uint16_t used_idx; 142 uint16_t id; 143 struct vring_packed_desc *desc; 144 uint16_t i; 145 146 desc = vq->vq_packed.ring.desc; 147 148 for (i = 0; i < num; i++) { 149 used_idx = vq->vq_used_cons_idx; 150 /* desc_is_used has a load-acquire or rte_io_rmb inside 151 * and wait for used desc in virtqueue. 152 */ 153 if (!desc_is_used(&desc[used_idx], vq)) 154 return i; 155 len[i] = desc[used_idx].len; 156 id = desc[used_idx].id; 157 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 158 if (unlikely(cookie == NULL)) { 159 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 160 vq->vq_used_cons_idx); 161 break; 162 } 163 rte_prefetch0(cookie); 164 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 165 rx_pkts[i] = cookie; 166 167 vq->vq_free_cnt++; 168 vq->vq_used_cons_idx++; 169 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 170 vq->vq_used_cons_idx -= vq->vq_nentries; 171 vq->vq_packed.used_wrap_counter ^= 1; 172 } 173 } 174 175 return i; 176 } 177 178 static uint16_t 179 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 180 uint32_t *len, uint16_t num) 181 { 182 struct vring_used_elem *uep; 183 struct rte_mbuf *cookie; 184 uint16_t used_idx, desc_idx; 185 uint16_t i; 186 187 /* Caller does the check */ 188 for (i = 0; i < num ; i++) { 189 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 190 uep = &vq->vq_split.ring.used->ring[used_idx]; 191 desc_idx = (uint16_t) uep->id; 192 len[i] = uep->len; 193 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 194 195 if (unlikely(cookie == NULL)) { 196 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 197 vq->vq_used_cons_idx); 198 break; 199 } 200 201 rte_prefetch0(cookie); 202 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 203 rx_pkts[i] = cookie; 204 vq->vq_used_cons_idx++; 205 vq_ring_free_chain(vq, desc_idx); 206 vq->vq_descx[desc_idx].cookie = NULL; 207 } 208 209 return i; 210 } 211 212 static uint16_t 213 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 214 struct rte_mbuf **rx_pkts, 215 uint32_t *len, 216 uint16_t num) 217 { 218 struct vring_used_elem *uep; 219 struct rte_mbuf *cookie; 220 uint16_t used_idx = 0; 221 uint16_t i; 222 223 if (unlikely(num == 0)) 224 return 0; 225 226 for (i = 0; i < num; i++) { 227 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 228 /* Desc idx same as used idx */ 229 uep = &vq->vq_split.ring.used->ring[used_idx]; 230 len[i] = uep->len; 231 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 232 233 if (unlikely(cookie == NULL)) { 234 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 235 vq->vq_used_cons_idx); 236 break; 237 } 238 239 rte_prefetch0(cookie); 240 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 241 rx_pkts[i] = cookie; 242 vq->vq_used_cons_idx++; 243 vq->vq_descx[used_idx].cookie = NULL; 244 } 245 246 vq_ring_free_inorder(vq, used_idx, i); 247 return i; 248 } 249 250 static inline int 251 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 252 struct rte_mbuf **cookies, 253 uint16_t num) 254 { 255 struct vq_desc_extra *dxp; 256 struct virtio_hw *hw = vq->hw; 257 struct vring_desc *start_dp; 258 uint16_t head_idx, idx, i = 0; 259 260 if (unlikely(vq->vq_free_cnt == 0)) 261 return -ENOSPC; 262 if (unlikely(vq->vq_free_cnt < num)) 263 return -EMSGSIZE; 264 265 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 266 start_dp = vq->vq_split.ring.desc; 267 268 while (i < num) { 269 idx = head_idx & (vq->vq_nentries - 1); 270 dxp = &vq->vq_descx[idx]; 271 dxp->cookie = (void *)cookies[i]; 272 dxp->ndescs = 1; 273 274 start_dp[idx].addr = cookies[i]->buf_iova + 275 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 276 start_dp[idx].len = cookies[i]->buf_len - 277 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 278 start_dp[idx].flags = VRING_DESC_F_WRITE; 279 280 vq_update_avail_ring(vq, idx); 281 head_idx++; 282 i++; 283 } 284 285 vq->vq_desc_head_idx += num; 286 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 287 return 0; 288 } 289 290 static inline int 291 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 292 uint16_t num) 293 { 294 struct vq_desc_extra *dxp; 295 struct virtio_hw *hw = vq->hw; 296 struct vring_desc *start_dp = vq->vq_split.ring.desc; 297 uint16_t idx, i; 298 299 if (unlikely(vq->vq_free_cnt == 0)) 300 return -ENOSPC; 301 if (unlikely(vq->vq_free_cnt < num)) 302 return -EMSGSIZE; 303 304 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 305 return -EFAULT; 306 307 for (i = 0; i < num; i++) { 308 idx = vq->vq_desc_head_idx; 309 dxp = &vq->vq_descx[idx]; 310 dxp->cookie = (void *)cookie[i]; 311 dxp->ndescs = 1; 312 313 start_dp[idx].addr = cookie[i]->buf_iova + 314 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 315 start_dp[idx].len = cookie[i]->buf_len - 316 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 317 start_dp[idx].flags = VRING_DESC_F_WRITE; 318 vq->vq_desc_head_idx = start_dp[idx].next; 319 vq_update_avail_ring(vq, idx); 320 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 321 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 322 break; 323 } 324 } 325 326 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 327 328 return 0; 329 } 330 331 static inline int 332 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 333 struct rte_mbuf **cookie, uint16_t num) 334 { 335 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 336 uint16_t flags = vq->vq_packed.cached_flags; 337 struct virtio_hw *hw = vq->hw; 338 struct vq_desc_extra *dxp; 339 uint16_t idx; 340 int i; 341 342 if (unlikely(vq->vq_free_cnt == 0)) 343 return -ENOSPC; 344 if (unlikely(vq->vq_free_cnt < num)) 345 return -EMSGSIZE; 346 347 for (i = 0; i < num; i++) { 348 idx = vq->vq_avail_idx; 349 dxp = &vq->vq_descx[idx]; 350 dxp->cookie = (void *)cookie[i]; 351 dxp->ndescs = 1; 352 353 start_dp[idx].addr = cookie[i]->buf_iova + 354 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 355 start_dp[idx].len = cookie[i]->buf_len - 356 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 357 358 vq->vq_desc_head_idx = dxp->next; 359 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 360 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 361 362 virtqueue_store_flags_packed(&start_dp[idx], flags, 363 hw->weak_barriers); 364 365 if (++vq->vq_avail_idx >= vq->vq_nentries) { 366 vq->vq_avail_idx -= vq->vq_nentries; 367 vq->vq_packed.cached_flags ^= 368 VRING_PACKED_DESC_F_AVAIL_USED; 369 flags = vq->vq_packed.cached_flags; 370 } 371 } 372 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 373 return 0; 374 } 375 376 /* When doing TSO, the IP length is not included in the pseudo header 377 * checksum of the packet given to the PMD, but for virtio it is 378 * expected. 379 */ 380 static void 381 virtio_tso_fix_cksum(struct rte_mbuf *m) 382 { 383 /* common case: header is not fragmented */ 384 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 385 m->l4_len)) { 386 struct rte_ipv4_hdr *iph; 387 struct rte_ipv6_hdr *ip6h; 388 struct rte_tcp_hdr *th; 389 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen; 390 uint32_t tmp; 391 392 iph = rte_pktmbuf_mtod_offset(m, 393 struct rte_ipv4_hdr *, m->l2_len); 394 th = RTE_PTR_ADD(iph, m->l3_len); 395 if ((iph->version_ihl >> 4) == 4) { 396 iph->hdr_checksum = 0; 397 iph->hdr_checksum = rte_ipv4_cksum(iph); 398 ip_len = iph->total_length; 399 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) - 400 m->l3_len); 401 } else { 402 ip6h = (struct rte_ipv6_hdr *)iph; 403 ip_paylen = ip6h->payload_len; 404 } 405 406 /* calculate the new phdr checksum not including ip_paylen */ 407 prev_cksum = th->cksum; 408 tmp = prev_cksum; 409 tmp += ip_paylen; 410 tmp = (tmp & 0xffff) + (tmp >> 16); 411 new_cksum = tmp; 412 413 /* replace it in the packet */ 414 th->cksum = new_cksum; 415 } 416 } 417 418 419 420 421 static inline void 422 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 423 struct rte_mbuf **cookies, 424 uint16_t num) 425 { 426 struct vq_desc_extra *dxp; 427 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 428 struct vring_desc *start_dp; 429 struct virtio_net_hdr *hdr; 430 uint16_t idx; 431 int16_t head_size = vq->hw->vtnet_hdr_size; 432 uint16_t i = 0; 433 434 idx = vq->vq_desc_head_idx; 435 start_dp = vq->vq_split.ring.desc; 436 437 while (i < num) { 438 idx = idx & (vq->vq_nentries - 1); 439 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 440 dxp->cookie = (void *)cookies[i]; 441 dxp->ndescs = 1; 442 virtio_update_packet_stats(&txvq->stats, cookies[i]); 443 444 hdr = rte_pktmbuf_mtod_offset(cookies[i], 445 struct virtio_net_hdr *, -head_size); 446 447 /* if offload disabled, hdr is not zeroed yet, do it now */ 448 if (!vq->hw->has_tx_offload) 449 virtqueue_clear_net_hdr(hdr); 450 else 451 virtqueue_xmit_offload(hdr, cookies[i], true); 452 453 start_dp[idx].addr = rte_mbuf_data_iova(cookies[i]) - head_size; 454 start_dp[idx].len = cookies[i]->data_len + head_size; 455 start_dp[idx].flags = 0; 456 457 458 vq_update_avail_ring(vq, idx); 459 460 idx++; 461 i++; 462 }; 463 464 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 465 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 466 } 467 468 static inline void 469 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 470 struct rte_mbuf *cookie, 471 int in_order) 472 { 473 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 474 struct vring_packed_desc *dp; 475 struct vq_desc_extra *dxp; 476 uint16_t idx, id, flags; 477 int16_t head_size = vq->hw->vtnet_hdr_size; 478 struct virtio_net_hdr *hdr; 479 480 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 481 idx = vq->vq_avail_idx; 482 dp = &vq->vq_packed.ring.desc[idx]; 483 484 dxp = &vq->vq_descx[id]; 485 dxp->ndescs = 1; 486 dxp->cookie = cookie; 487 488 flags = vq->vq_packed.cached_flags; 489 490 /* prepend cannot fail, checked by caller */ 491 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 492 -head_size); 493 494 /* if offload disabled, hdr is not zeroed yet, do it now */ 495 if (!vq->hw->has_tx_offload) 496 virtqueue_clear_net_hdr(hdr); 497 else 498 virtqueue_xmit_offload(hdr, cookie, true); 499 500 dp->addr = rte_mbuf_data_iova(cookie) - head_size; 501 dp->len = cookie->data_len + head_size; 502 dp->id = id; 503 504 if (++vq->vq_avail_idx >= vq->vq_nentries) { 505 vq->vq_avail_idx -= vq->vq_nentries; 506 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 507 } 508 509 vq->vq_free_cnt--; 510 511 if (!in_order) { 512 vq->vq_desc_head_idx = dxp->next; 513 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 514 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 515 } 516 517 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 518 } 519 520 static inline void 521 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 522 uint16_t needed, int use_indirect, int can_push, 523 int in_order) 524 { 525 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 526 struct vq_desc_extra *dxp; 527 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 528 struct vring_desc *start_dp; 529 uint16_t seg_num = cookie->nb_segs; 530 uint16_t head_idx, idx; 531 int16_t head_size = vq->hw->vtnet_hdr_size; 532 bool prepend_header = false; 533 struct virtio_net_hdr *hdr; 534 535 head_idx = vq->vq_desc_head_idx; 536 idx = head_idx; 537 if (in_order) 538 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 539 else 540 dxp = &vq->vq_descx[idx]; 541 dxp->cookie = (void *)cookie; 542 dxp->ndescs = needed; 543 544 start_dp = vq->vq_split.ring.desc; 545 546 if (can_push) { 547 /* prepend cannot fail, checked by caller */ 548 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 549 -head_size); 550 prepend_header = true; 551 552 /* if offload disabled, it is not zeroed below, do it now */ 553 if (!vq->hw->has_tx_offload) 554 virtqueue_clear_net_hdr(hdr); 555 } else if (use_indirect) { 556 /* setup tx ring slot to point to indirect 557 * descriptor list stored in reserved region. 558 * 559 * the first slot in indirect ring is already preset 560 * to point to the header in reserved region 561 */ 562 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 563 RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 564 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 565 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 566 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 567 568 /* loop below will fill in rest of the indirect elements */ 569 start_dp = txr[idx].tx_indir; 570 idx = 1; 571 } else { 572 /* setup first tx ring slot to point to header 573 * stored in reserved region. 574 */ 575 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 576 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 577 start_dp[idx].len = vq->hw->vtnet_hdr_size; 578 start_dp[idx].flags = VRING_DESC_F_NEXT; 579 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 580 581 idx = start_dp[idx].next; 582 } 583 584 virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload); 585 586 do { 587 start_dp[idx].addr = rte_mbuf_data_iova(cookie); 588 start_dp[idx].len = cookie->data_len; 589 if (prepend_header) { 590 start_dp[idx].addr -= head_size; 591 start_dp[idx].len += head_size; 592 prepend_header = false; 593 } 594 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 595 idx = start_dp[idx].next; 596 } while ((cookie = cookie->next) != NULL); 597 598 if (use_indirect) 599 idx = vq->vq_split.ring.desc[head_idx].next; 600 601 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 602 603 vq->vq_desc_head_idx = idx; 604 vq_update_avail_ring(vq, head_idx); 605 606 if (!in_order) { 607 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 608 vq->vq_desc_tail_idx = idx; 609 } 610 } 611 612 void 613 virtio_dev_cq_start(struct rte_eth_dev *dev) 614 { 615 struct virtio_hw *hw = dev->data->dev_private; 616 617 if (hw->cvq) { 618 rte_spinlock_init(&hw->cvq->lock); 619 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); 620 } 621 } 622 623 int 624 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 625 uint16_t queue_idx, 626 uint16_t nb_desc, 627 unsigned int socket_id __rte_unused, 628 const struct rte_eth_rxconf *rx_conf, 629 struct rte_mempool *mp) 630 { 631 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 632 struct virtio_hw *hw = dev->data->dev_private; 633 struct virtqueue *vq = hw->vqs[vq_idx]; 634 struct virtnet_rx *rxvq; 635 uint16_t rx_free_thresh; 636 637 PMD_INIT_FUNC_TRACE(); 638 639 if (rx_conf->rx_deferred_start) { 640 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 641 return -EINVAL; 642 } 643 644 rx_free_thresh = rx_conf->rx_free_thresh; 645 if (rx_free_thresh == 0) 646 rx_free_thresh = 647 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); 648 649 if (rx_free_thresh & 0x3) { 650 RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four." 651 " (rx_free_thresh=%u port=%u queue=%u)\n", 652 rx_free_thresh, dev->data->port_id, queue_idx); 653 return -EINVAL; 654 } 655 656 if (rx_free_thresh >= vq->vq_nentries) { 657 RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the " 658 "number of RX entries (%u)." 659 " (rx_free_thresh=%u port=%u queue=%u)\n", 660 vq->vq_nentries, 661 rx_free_thresh, dev->data->port_id, queue_idx); 662 return -EINVAL; 663 } 664 vq->vq_free_thresh = rx_free_thresh; 665 666 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 667 nb_desc = vq->vq_nentries; 668 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 669 670 rxvq = &vq->rxq; 671 rxvq->queue_id = queue_idx; 672 rxvq->mpool = mp; 673 dev->data->rx_queues[queue_idx] = rxvq; 674 675 return 0; 676 } 677 678 int 679 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 680 { 681 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 682 struct virtio_hw *hw = dev->data->dev_private; 683 struct virtqueue *vq = hw->vqs[vq_idx]; 684 struct virtnet_rx *rxvq = &vq->rxq; 685 struct rte_mbuf *m; 686 uint16_t desc_idx; 687 int error, nbufs, i; 688 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 689 690 PMD_INIT_FUNC_TRACE(); 691 692 /* Allocate blank mbufs for the each rx descriptor */ 693 nbufs = 0; 694 695 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 696 for (desc_idx = 0; desc_idx < vq->vq_nentries; 697 desc_idx++) { 698 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 699 vq->vq_split.ring.desc[desc_idx].flags = 700 VRING_DESC_F_WRITE; 701 } 702 703 virtio_rxq_vec_setup(rxvq); 704 } 705 706 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); 707 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) 708 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; 709 710 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 711 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 712 virtio_rxq_rearm_vec(rxvq); 713 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 714 } 715 } else if (!virtio_with_packed_queue(vq->hw) && in_order) { 716 if ((!virtqueue_full(vq))) { 717 uint16_t free_cnt = vq->vq_free_cnt; 718 struct rte_mbuf *pkts[free_cnt]; 719 720 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 721 free_cnt)) { 722 error = virtqueue_enqueue_refill_inorder(vq, 723 pkts, 724 free_cnt); 725 if (unlikely(error)) { 726 for (i = 0; i < free_cnt; i++) 727 rte_pktmbuf_free(pkts[i]); 728 } 729 } 730 731 nbufs += free_cnt; 732 vq_update_avail_idx(vq); 733 } 734 } else { 735 while (!virtqueue_full(vq)) { 736 m = rte_mbuf_raw_alloc(rxvq->mpool); 737 if (m == NULL) 738 break; 739 740 /* Enqueue allocated buffers */ 741 if (virtio_with_packed_queue(vq->hw)) 742 error = virtqueue_enqueue_recv_refill_packed(vq, 743 &m, 1); 744 else 745 error = virtqueue_enqueue_recv_refill(vq, 746 &m, 1); 747 if (error) { 748 rte_pktmbuf_free(m); 749 break; 750 } 751 nbufs++; 752 } 753 754 if (!virtio_with_packed_queue(vq->hw)) 755 vq_update_avail_idx(vq); 756 } 757 758 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); 759 760 VIRTQUEUE_DUMP(vq); 761 762 return 0; 763 } 764 765 /* 766 * struct rte_eth_dev *dev: Used to update dev 767 * uint16_t nb_desc: Defaults to values read from config space 768 * unsigned int socket_id: Used to allocate memzone 769 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 770 * uint16_t queue_idx: Just used as an index in dev txq list 771 */ 772 int 773 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 774 uint16_t queue_idx, 775 uint16_t nb_desc, 776 unsigned int socket_id __rte_unused, 777 const struct rte_eth_txconf *tx_conf) 778 { 779 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 780 struct virtio_hw *hw = dev->data->dev_private; 781 struct virtqueue *vq = hw->vqs[vq_idx]; 782 struct virtnet_tx *txvq; 783 uint16_t tx_free_thresh; 784 785 PMD_INIT_FUNC_TRACE(); 786 787 if (tx_conf->tx_deferred_start) { 788 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 789 return -EINVAL; 790 } 791 792 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 793 nb_desc = vq->vq_nentries; 794 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 795 796 txvq = &vq->txq; 797 txvq->queue_id = queue_idx; 798 799 tx_free_thresh = tx_conf->tx_free_thresh; 800 if (tx_free_thresh == 0) 801 tx_free_thresh = 802 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 803 804 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 805 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " 806 "number of TX entries minus 3 (%u)." 807 " (tx_free_thresh=%u port=%u queue=%u)\n", 808 vq->vq_nentries - 3, 809 tx_free_thresh, dev->data->port_id, queue_idx); 810 return -EINVAL; 811 } 812 813 vq->vq_free_thresh = tx_free_thresh; 814 815 dev->data->tx_queues[queue_idx] = txvq; 816 return 0; 817 } 818 819 int 820 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 821 uint16_t queue_idx) 822 { 823 uint8_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 824 struct virtio_hw *hw = dev->data->dev_private; 825 struct virtqueue *vq = hw->vqs[vq_idx]; 826 827 PMD_INIT_FUNC_TRACE(); 828 829 if (!virtio_with_packed_queue(hw)) { 830 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 831 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 832 } 833 834 VIRTQUEUE_DUMP(vq); 835 836 return 0; 837 } 838 839 static inline void 840 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 841 { 842 int error; 843 /* 844 * Requeue the discarded mbuf. This should always be 845 * successful since it was just dequeued. 846 */ 847 if (virtio_with_packed_queue(vq->hw)) 848 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 849 else 850 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 851 852 if (unlikely(error)) { 853 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 854 rte_pktmbuf_free(m); 855 } 856 } 857 858 static inline void 859 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 860 { 861 int error; 862 863 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 864 if (unlikely(error)) { 865 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 866 rte_pktmbuf_free(m); 867 } 868 } 869 870 /* Optionally fill offload information in structure */ 871 static inline int 872 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 873 { 874 struct rte_net_hdr_lens hdr_lens; 875 uint32_t hdrlen, ptype; 876 int l4_supported = 0; 877 878 /* nothing to do */ 879 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 880 return 0; 881 882 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; 883 884 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 885 m->packet_type = ptype; 886 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 887 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 888 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 889 l4_supported = 1; 890 891 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 892 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 893 if (hdr->csum_start <= hdrlen && l4_supported) { 894 m->ol_flags |= PKT_RX_L4_CKSUM_NONE; 895 } else { 896 /* Unknown proto or tunnel, do sw cksum. We can assume 897 * the cksum field is in the first segment since the 898 * buffers we provided to the host are large enough. 899 * In case of SCTP, this will be wrong since it's a CRC 900 * but there's nothing we can do. 901 */ 902 uint16_t csum = 0, off; 903 904 if (rte_raw_cksum_mbuf(m, hdr->csum_start, 905 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 906 &csum) < 0) 907 return -EINVAL; 908 if (likely(csum != 0xffff)) 909 csum = ~csum; 910 off = hdr->csum_offset + hdr->csum_start; 911 if (rte_pktmbuf_data_len(m) >= off + 1) 912 *rte_pktmbuf_mtod_offset(m, uint16_t *, 913 off) = csum; 914 } 915 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 916 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 917 } 918 919 /* GSO request, save required information in mbuf */ 920 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 921 /* Check unsupported modes */ 922 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 923 (hdr->gso_size == 0)) { 924 return -EINVAL; 925 } 926 927 /* Update mss lengthes in mbuf */ 928 m->tso_segsz = hdr->gso_size; 929 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 930 case VIRTIO_NET_HDR_GSO_TCPV4: 931 case VIRTIO_NET_HDR_GSO_TCPV6: 932 m->ol_flags |= PKT_RX_LRO | \ 933 PKT_RX_L4_CKSUM_NONE; 934 break; 935 default: 936 return -EINVAL; 937 } 938 } 939 940 return 0; 941 } 942 943 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 944 uint16_t 945 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 946 { 947 struct virtnet_rx *rxvq = rx_queue; 948 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 949 struct virtio_hw *hw = vq->hw; 950 struct rte_mbuf *rxm; 951 uint16_t nb_used, num, nb_rx; 952 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 953 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 954 int error; 955 uint32_t i, nb_enqueued; 956 uint32_t hdr_size; 957 struct virtio_net_hdr *hdr; 958 959 nb_rx = 0; 960 if (unlikely(hw->started == 0)) 961 return nb_rx; 962 963 nb_used = virtqueue_nused(vq); 964 965 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 966 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 967 num = VIRTIO_MBUF_BURST_SZ; 968 if (likely(num > DESC_PER_CACHELINE)) 969 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 970 971 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 972 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 973 974 nb_enqueued = 0; 975 hdr_size = hw->vtnet_hdr_size; 976 977 for (i = 0; i < num ; i++) { 978 rxm = rcv_pkts[i]; 979 980 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 981 982 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 983 PMD_RX_LOG(ERR, "Packet drop"); 984 nb_enqueued++; 985 virtio_discard_rxbuf(vq, rxm); 986 rxvq->stats.errors++; 987 continue; 988 } 989 990 rxm->port = rxvq->port_id; 991 rxm->data_off = RTE_PKTMBUF_HEADROOM; 992 rxm->ol_flags = 0; 993 rxm->vlan_tci = 0; 994 995 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 996 rxm->data_len = (uint16_t)(len[i] - hdr_size); 997 998 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 999 RTE_PKTMBUF_HEADROOM - hdr_size); 1000 1001 if (hw->vlan_strip) 1002 rte_vlan_strip(rxm); 1003 1004 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1005 virtio_discard_rxbuf(vq, rxm); 1006 rxvq->stats.errors++; 1007 continue; 1008 } 1009 1010 virtio_rx_stats_updated(rxvq, rxm); 1011 1012 rx_pkts[nb_rx++] = rxm; 1013 } 1014 1015 rxvq->stats.packets += nb_rx; 1016 1017 /* Allocate new mbuf for the used descriptor */ 1018 if (likely(!virtqueue_full(vq))) { 1019 uint16_t free_cnt = vq->vq_free_cnt; 1020 struct rte_mbuf *new_pkts[free_cnt]; 1021 1022 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1023 free_cnt) == 0)) { 1024 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1025 free_cnt); 1026 if (unlikely(error)) { 1027 for (i = 0; i < free_cnt; i++) 1028 rte_pktmbuf_free(new_pkts[i]); 1029 } 1030 nb_enqueued += free_cnt; 1031 } else { 1032 struct rte_eth_dev *dev = 1033 &rte_eth_devices[rxvq->port_id]; 1034 dev->data->rx_mbuf_alloc_failed += free_cnt; 1035 } 1036 } 1037 1038 if (likely(nb_enqueued)) { 1039 vq_update_avail_idx(vq); 1040 1041 if (unlikely(virtqueue_kick_prepare(vq))) { 1042 virtqueue_notify(vq); 1043 PMD_RX_LOG(DEBUG, "Notified"); 1044 } 1045 } 1046 1047 return nb_rx; 1048 } 1049 1050 uint16_t 1051 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1052 uint16_t nb_pkts) 1053 { 1054 struct virtnet_rx *rxvq = rx_queue; 1055 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1056 struct virtio_hw *hw = vq->hw; 1057 struct rte_mbuf *rxm; 1058 uint16_t num, nb_rx; 1059 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1060 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1061 int error; 1062 uint32_t i, nb_enqueued; 1063 uint32_t hdr_size; 1064 struct virtio_net_hdr *hdr; 1065 1066 nb_rx = 0; 1067 if (unlikely(hw->started == 0)) 1068 return nb_rx; 1069 1070 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1071 if (likely(num > DESC_PER_CACHELINE)) 1072 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1073 1074 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1075 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1076 1077 nb_enqueued = 0; 1078 hdr_size = hw->vtnet_hdr_size; 1079 1080 for (i = 0; i < num; i++) { 1081 rxm = rcv_pkts[i]; 1082 1083 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1084 1085 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1086 PMD_RX_LOG(ERR, "Packet drop"); 1087 nb_enqueued++; 1088 virtio_discard_rxbuf(vq, rxm); 1089 rxvq->stats.errors++; 1090 continue; 1091 } 1092 1093 rxm->port = rxvq->port_id; 1094 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1095 rxm->ol_flags = 0; 1096 rxm->vlan_tci = 0; 1097 1098 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1099 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1100 1101 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1102 RTE_PKTMBUF_HEADROOM - hdr_size); 1103 1104 if (hw->vlan_strip) 1105 rte_vlan_strip(rxm); 1106 1107 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1108 virtio_discard_rxbuf(vq, rxm); 1109 rxvq->stats.errors++; 1110 continue; 1111 } 1112 1113 virtio_rx_stats_updated(rxvq, rxm); 1114 1115 rx_pkts[nb_rx++] = rxm; 1116 } 1117 1118 rxvq->stats.packets += nb_rx; 1119 1120 /* Allocate new mbuf for the used descriptor */ 1121 if (likely(!virtqueue_full(vq))) { 1122 uint16_t free_cnt = vq->vq_free_cnt; 1123 struct rte_mbuf *new_pkts[free_cnt]; 1124 1125 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1126 free_cnt) == 0)) { 1127 error = virtqueue_enqueue_recv_refill_packed(vq, 1128 new_pkts, free_cnt); 1129 if (unlikely(error)) { 1130 for (i = 0; i < free_cnt; i++) 1131 rte_pktmbuf_free(new_pkts[i]); 1132 } 1133 nb_enqueued += free_cnt; 1134 } else { 1135 struct rte_eth_dev *dev = 1136 &rte_eth_devices[rxvq->port_id]; 1137 dev->data->rx_mbuf_alloc_failed += free_cnt; 1138 } 1139 } 1140 1141 if (likely(nb_enqueued)) { 1142 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1143 virtqueue_notify(vq); 1144 PMD_RX_LOG(DEBUG, "Notified"); 1145 } 1146 } 1147 1148 return nb_rx; 1149 } 1150 1151 1152 uint16_t 1153 virtio_recv_pkts_inorder(void *rx_queue, 1154 struct rte_mbuf **rx_pkts, 1155 uint16_t nb_pkts) 1156 { 1157 struct virtnet_rx *rxvq = rx_queue; 1158 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1159 struct virtio_hw *hw = vq->hw; 1160 struct rte_mbuf *rxm; 1161 struct rte_mbuf *prev = NULL; 1162 uint16_t nb_used, num, nb_rx; 1163 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1164 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1165 int error; 1166 uint32_t nb_enqueued; 1167 uint32_t seg_num; 1168 uint32_t seg_res; 1169 uint32_t hdr_size; 1170 int32_t i; 1171 1172 nb_rx = 0; 1173 if (unlikely(hw->started == 0)) 1174 return nb_rx; 1175 1176 nb_used = virtqueue_nused(vq); 1177 nb_used = RTE_MIN(nb_used, nb_pkts); 1178 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1179 1180 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1181 1182 nb_enqueued = 0; 1183 seg_num = 1; 1184 seg_res = 0; 1185 hdr_size = hw->vtnet_hdr_size; 1186 1187 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1188 1189 for (i = 0; i < num; i++) { 1190 struct virtio_net_hdr_mrg_rxbuf *header; 1191 1192 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1193 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1194 1195 rxm = rcv_pkts[i]; 1196 1197 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1198 PMD_RX_LOG(ERR, "Packet drop"); 1199 nb_enqueued++; 1200 virtio_discard_rxbuf_inorder(vq, rxm); 1201 rxvq->stats.errors++; 1202 continue; 1203 } 1204 1205 header = (struct virtio_net_hdr_mrg_rxbuf *) 1206 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1207 - hdr_size); 1208 1209 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1210 seg_num = header->num_buffers; 1211 if (seg_num == 0) 1212 seg_num = 1; 1213 } else { 1214 seg_num = 1; 1215 } 1216 1217 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1218 rxm->nb_segs = seg_num; 1219 rxm->ol_flags = 0; 1220 rxm->vlan_tci = 0; 1221 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1222 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1223 1224 rxm->port = rxvq->port_id; 1225 1226 rx_pkts[nb_rx] = rxm; 1227 prev = rxm; 1228 1229 if (vq->hw->has_rx_offload && 1230 virtio_rx_offload(rxm, &header->hdr) < 0) { 1231 virtio_discard_rxbuf_inorder(vq, rxm); 1232 rxvq->stats.errors++; 1233 continue; 1234 } 1235 1236 if (hw->vlan_strip) 1237 rte_vlan_strip(rx_pkts[nb_rx]); 1238 1239 seg_res = seg_num - 1; 1240 1241 /* Merge remaining segments */ 1242 while (seg_res != 0 && i < (num - 1)) { 1243 i++; 1244 1245 rxm = rcv_pkts[i]; 1246 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1247 rxm->pkt_len = (uint32_t)(len[i]); 1248 rxm->data_len = (uint16_t)(len[i]); 1249 1250 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1251 1252 prev->next = rxm; 1253 prev = rxm; 1254 seg_res -= 1; 1255 } 1256 1257 if (!seg_res) { 1258 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1259 nb_rx++; 1260 } 1261 } 1262 1263 /* Last packet still need merge segments */ 1264 while (seg_res != 0) { 1265 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1266 VIRTIO_MBUF_BURST_SZ); 1267 1268 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1269 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1270 rcv_cnt); 1271 uint16_t extra_idx = 0; 1272 1273 rcv_cnt = num; 1274 while (extra_idx < rcv_cnt) { 1275 rxm = rcv_pkts[extra_idx]; 1276 rxm->data_off = 1277 RTE_PKTMBUF_HEADROOM - hdr_size; 1278 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1279 rxm->data_len = (uint16_t)(len[extra_idx]); 1280 prev->next = rxm; 1281 prev = rxm; 1282 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1283 extra_idx += 1; 1284 }; 1285 seg_res -= rcv_cnt; 1286 1287 if (!seg_res) { 1288 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1289 nb_rx++; 1290 } 1291 } else { 1292 PMD_RX_LOG(ERR, 1293 "No enough segments for packet."); 1294 rte_pktmbuf_free(rx_pkts[nb_rx]); 1295 rxvq->stats.errors++; 1296 break; 1297 } 1298 } 1299 1300 rxvq->stats.packets += nb_rx; 1301 1302 /* Allocate new mbuf for the used descriptor */ 1303 1304 if (likely(!virtqueue_full(vq))) { 1305 /* free_cnt may include mrg descs */ 1306 uint16_t free_cnt = vq->vq_free_cnt; 1307 struct rte_mbuf *new_pkts[free_cnt]; 1308 1309 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1310 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1311 free_cnt); 1312 if (unlikely(error)) { 1313 for (i = 0; i < free_cnt; i++) 1314 rte_pktmbuf_free(new_pkts[i]); 1315 } 1316 nb_enqueued += free_cnt; 1317 } else { 1318 struct rte_eth_dev *dev = 1319 &rte_eth_devices[rxvq->port_id]; 1320 dev->data->rx_mbuf_alloc_failed += free_cnt; 1321 } 1322 } 1323 1324 if (likely(nb_enqueued)) { 1325 vq_update_avail_idx(vq); 1326 1327 if (unlikely(virtqueue_kick_prepare(vq))) { 1328 virtqueue_notify(vq); 1329 PMD_RX_LOG(DEBUG, "Notified"); 1330 } 1331 } 1332 1333 return nb_rx; 1334 } 1335 1336 uint16_t 1337 virtio_recv_mergeable_pkts(void *rx_queue, 1338 struct rte_mbuf **rx_pkts, 1339 uint16_t nb_pkts) 1340 { 1341 struct virtnet_rx *rxvq = rx_queue; 1342 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1343 struct virtio_hw *hw = vq->hw; 1344 struct rte_mbuf *rxm; 1345 struct rte_mbuf *prev = NULL; 1346 uint16_t nb_used, num, nb_rx = 0; 1347 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1348 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1349 int error; 1350 uint32_t nb_enqueued = 0; 1351 uint32_t seg_num = 0; 1352 uint32_t seg_res = 0; 1353 uint32_t hdr_size = hw->vtnet_hdr_size; 1354 int32_t i; 1355 1356 if (unlikely(hw->started == 0)) 1357 return nb_rx; 1358 1359 nb_used = virtqueue_nused(vq); 1360 1361 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1362 1363 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1364 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1365 num = VIRTIO_MBUF_BURST_SZ; 1366 if (likely(num > DESC_PER_CACHELINE)) 1367 num = num - ((vq->vq_used_cons_idx + num) % 1368 DESC_PER_CACHELINE); 1369 1370 1371 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1372 1373 for (i = 0; i < num; i++) { 1374 struct virtio_net_hdr_mrg_rxbuf *header; 1375 1376 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1377 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1378 1379 rxm = rcv_pkts[i]; 1380 1381 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1382 PMD_RX_LOG(ERR, "Packet drop"); 1383 nb_enqueued++; 1384 virtio_discard_rxbuf(vq, rxm); 1385 rxvq->stats.errors++; 1386 continue; 1387 } 1388 1389 header = (struct virtio_net_hdr_mrg_rxbuf *) 1390 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1391 - hdr_size); 1392 seg_num = header->num_buffers; 1393 if (seg_num == 0) 1394 seg_num = 1; 1395 1396 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1397 rxm->nb_segs = seg_num; 1398 rxm->ol_flags = 0; 1399 rxm->vlan_tci = 0; 1400 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1401 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1402 1403 rxm->port = rxvq->port_id; 1404 1405 rx_pkts[nb_rx] = rxm; 1406 prev = rxm; 1407 1408 if (hw->has_rx_offload && 1409 virtio_rx_offload(rxm, &header->hdr) < 0) { 1410 virtio_discard_rxbuf(vq, rxm); 1411 rxvq->stats.errors++; 1412 continue; 1413 } 1414 1415 if (hw->vlan_strip) 1416 rte_vlan_strip(rx_pkts[nb_rx]); 1417 1418 seg_res = seg_num - 1; 1419 1420 /* Merge remaining segments */ 1421 while (seg_res != 0 && i < (num - 1)) { 1422 i++; 1423 1424 rxm = rcv_pkts[i]; 1425 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1426 rxm->pkt_len = (uint32_t)(len[i]); 1427 rxm->data_len = (uint16_t)(len[i]); 1428 1429 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1430 1431 prev->next = rxm; 1432 prev = rxm; 1433 seg_res -= 1; 1434 } 1435 1436 if (!seg_res) { 1437 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1438 nb_rx++; 1439 } 1440 } 1441 1442 /* Last packet still need merge segments */ 1443 while (seg_res != 0) { 1444 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1445 VIRTIO_MBUF_BURST_SZ); 1446 1447 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1448 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1449 rcv_cnt); 1450 uint16_t extra_idx = 0; 1451 1452 rcv_cnt = num; 1453 while (extra_idx < rcv_cnt) { 1454 rxm = rcv_pkts[extra_idx]; 1455 rxm->data_off = 1456 RTE_PKTMBUF_HEADROOM - hdr_size; 1457 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1458 rxm->data_len = (uint16_t)(len[extra_idx]); 1459 prev->next = rxm; 1460 prev = rxm; 1461 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1462 extra_idx += 1; 1463 }; 1464 seg_res -= rcv_cnt; 1465 1466 if (!seg_res) { 1467 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1468 nb_rx++; 1469 } 1470 } else { 1471 PMD_RX_LOG(ERR, 1472 "No enough segments for packet."); 1473 rte_pktmbuf_free(rx_pkts[nb_rx]); 1474 rxvq->stats.errors++; 1475 break; 1476 } 1477 } 1478 1479 rxvq->stats.packets += nb_rx; 1480 1481 /* Allocate new mbuf for the used descriptor */ 1482 if (likely(!virtqueue_full(vq))) { 1483 /* free_cnt may include mrg descs */ 1484 uint16_t free_cnt = vq->vq_free_cnt; 1485 struct rte_mbuf *new_pkts[free_cnt]; 1486 1487 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1488 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1489 free_cnt); 1490 if (unlikely(error)) { 1491 for (i = 0; i < free_cnt; i++) 1492 rte_pktmbuf_free(new_pkts[i]); 1493 } 1494 nb_enqueued += free_cnt; 1495 } else { 1496 struct rte_eth_dev *dev = 1497 &rte_eth_devices[rxvq->port_id]; 1498 dev->data->rx_mbuf_alloc_failed += free_cnt; 1499 } 1500 } 1501 1502 if (likely(nb_enqueued)) { 1503 vq_update_avail_idx(vq); 1504 1505 if (unlikely(virtqueue_kick_prepare(vq))) { 1506 virtqueue_notify(vq); 1507 PMD_RX_LOG(DEBUG, "Notified"); 1508 } 1509 } 1510 1511 return nb_rx; 1512 } 1513 1514 uint16_t 1515 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1516 struct rte_mbuf **rx_pkts, 1517 uint16_t nb_pkts) 1518 { 1519 struct virtnet_rx *rxvq = rx_queue; 1520 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1521 struct virtio_hw *hw = vq->hw; 1522 struct rte_mbuf *rxm; 1523 struct rte_mbuf *prev = NULL; 1524 uint16_t num, nb_rx = 0; 1525 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1526 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1527 uint32_t nb_enqueued = 0; 1528 uint32_t seg_num = 0; 1529 uint32_t seg_res = 0; 1530 uint32_t hdr_size = hw->vtnet_hdr_size; 1531 int32_t i; 1532 int error; 1533 1534 if (unlikely(hw->started == 0)) 1535 return nb_rx; 1536 1537 1538 num = nb_pkts; 1539 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1540 num = VIRTIO_MBUF_BURST_SZ; 1541 if (likely(num > DESC_PER_CACHELINE)) 1542 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1543 1544 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1545 1546 for (i = 0; i < num; i++) { 1547 struct virtio_net_hdr_mrg_rxbuf *header; 1548 1549 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1550 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1551 1552 rxm = rcv_pkts[i]; 1553 1554 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1555 PMD_RX_LOG(ERR, "Packet drop"); 1556 nb_enqueued++; 1557 virtio_discard_rxbuf(vq, rxm); 1558 rxvq->stats.errors++; 1559 continue; 1560 } 1561 1562 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1563 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1564 seg_num = header->num_buffers; 1565 1566 if (seg_num == 0) 1567 seg_num = 1; 1568 1569 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1570 rxm->nb_segs = seg_num; 1571 rxm->ol_flags = 0; 1572 rxm->vlan_tci = 0; 1573 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1574 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1575 1576 rxm->port = rxvq->port_id; 1577 rx_pkts[nb_rx] = rxm; 1578 prev = rxm; 1579 1580 if (hw->has_rx_offload && 1581 virtio_rx_offload(rxm, &header->hdr) < 0) { 1582 virtio_discard_rxbuf(vq, rxm); 1583 rxvq->stats.errors++; 1584 continue; 1585 } 1586 1587 if (hw->vlan_strip) 1588 rte_vlan_strip(rx_pkts[nb_rx]); 1589 1590 seg_res = seg_num - 1; 1591 1592 /* Merge remaining segments */ 1593 while (seg_res != 0 && i < (num - 1)) { 1594 i++; 1595 1596 rxm = rcv_pkts[i]; 1597 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1598 rxm->pkt_len = (uint32_t)(len[i]); 1599 rxm->data_len = (uint16_t)(len[i]); 1600 1601 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1602 1603 prev->next = rxm; 1604 prev = rxm; 1605 seg_res -= 1; 1606 } 1607 1608 if (!seg_res) { 1609 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1610 nb_rx++; 1611 } 1612 } 1613 1614 /* Last packet still need merge segments */ 1615 while (seg_res != 0) { 1616 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1617 VIRTIO_MBUF_BURST_SZ); 1618 uint16_t extra_idx = 0; 1619 1620 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1621 len, rcv_cnt); 1622 if (unlikely(rcv_cnt == 0)) { 1623 PMD_RX_LOG(ERR, "No enough segments for packet."); 1624 rte_pktmbuf_free(rx_pkts[nb_rx]); 1625 rxvq->stats.errors++; 1626 break; 1627 } 1628 1629 while (extra_idx < rcv_cnt) { 1630 rxm = rcv_pkts[extra_idx]; 1631 1632 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1633 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1634 rxm->data_len = (uint16_t)(len[extra_idx]); 1635 1636 prev->next = rxm; 1637 prev = rxm; 1638 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1639 extra_idx += 1; 1640 } 1641 seg_res -= rcv_cnt; 1642 if (!seg_res) { 1643 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1644 nb_rx++; 1645 } 1646 } 1647 1648 rxvq->stats.packets += nb_rx; 1649 1650 /* Allocate new mbuf for the used descriptor */ 1651 if (likely(!virtqueue_full(vq))) { 1652 /* free_cnt may include mrg descs */ 1653 uint16_t free_cnt = vq->vq_free_cnt; 1654 struct rte_mbuf *new_pkts[free_cnt]; 1655 1656 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1657 error = virtqueue_enqueue_recv_refill_packed(vq, 1658 new_pkts, free_cnt); 1659 if (unlikely(error)) { 1660 for (i = 0; i < free_cnt; i++) 1661 rte_pktmbuf_free(new_pkts[i]); 1662 } 1663 nb_enqueued += free_cnt; 1664 } else { 1665 struct rte_eth_dev *dev = 1666 &rte_eth_devices[rxvq->port_id]; 1667 dev->data->rx_mbuf_alloc_failed += free_cnt; 1668 } 1669 } 1670 1671 if (likely(nb_enqueued)) { 1672 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1673 virtqueue_notify(vq); 1674 PMD_RX_LOG(DEBUG, "Notified"); 1675 } 1676 } 1677 1678 return nb_rx; 1679 } 1680 1681 uint16_t 1682 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1683 uint16_t nb_pkts) 1684 { 1685 uint16_t nb_tx; 1686 int error; 1687 1688 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1689 struct rte_mbuf *m = tx_pkts[nb_tx]; 1690 1691 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1692 error = rte_validate_tx_offload(m); 1693 if (unlikely(error)) { 1694 rte_errno = -error; 1695 break; 1696 } 1697 #endif 1698 1699 /* Do VLAN tag insertion */ 1700 if (unlikely(m->ol_flags & PKT_TX_VLAN_PKT)) { 1701 error = rte_vlan_insert(&m); 1702 /* rte_vlan_insert() may change pointer 1703 * even in the case of failure 1704 */ 1705 tx_pkts[nb_tx] = m; 1706 1707 if (unlikely(error)) { 1708 rte_errno = -error; 1709 break; 1710 } 1711 } 1712 1713 error = rte_net_intel_cksum_prepare(m); 1714 if (unlikely(error)) { 1715 rte_errno = -error; 1716 break; 1717 } 1718 1719 if (m->ol_flags & PKT_TX_TCP_SEG) 1720 virtio_tso_fix_cksum(m); 1721 } 1722 1723 return nb_tx; 1724 } 1725 1726 uint16_t 1727 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 1728 uint16_t nb_pkts) 1729 { 1730 struct virtnet_tx *txvq = tx_queue; 1731 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1732 struct virtio_hw *hw = vq->hw; 1733 uint16_t hdr_size = hw->vtnet_hdr_size; 1734 uint16_t nb_tx = 0; 1735 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 1736 1737 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1738 return nb_tx; 1739 1740 if (unlikely(nb_pkts < 1)) 1741 return nb_pkts; 1742 1743 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1744 1745 if (nb_pkts > vq->vq_free_cnt) 1746 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 1747 in_order); 1748 1749 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1750 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1751 int can_push = 0, use_indirect = 0, slots, need; 1752 1753 /* optimize ring usage */ 1754 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1755 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1756 rte_mbuf_refcnt_read(txm) == 1 && 1757 RTE_MBUF_DIRECT(txm) && 1758 txm->nb_segs == 1 && 1759 rte_pktmbuf_headroom(txm) >= hdr_size && 1760 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1761 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1762 can_push = 1; 1763 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1764 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1765 use_indirect = 1; 1766 /* How many main ring entries are needed to this Tx? 1767 * indirect => 1 1768 * any_layout => number of segments 1769 * default => number of segments + 1 1770 */ 1771 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1772 need = slots - vq->vq_free_cnt; 1773 1774 /* Positive value indicates it need free vring descriptors */ 1775 if (unlikely(need > 0)) { 1776 virtio_xmit_cleanup_packed(vq, need, in_order); 1777 need = slots - vq->vq_free_cnt; 1778 if (unlikely(need > 0)) { 1779 PMD_TX_LOG(ERR, 1780 "No free tx descriptors to transmit"); 1781 break; 1782 } 1783 } 1784 1785 /* Enqueue Packet buffers */ 1786 if (can_push) 1787 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 1788 else 1789 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 1790 use_indirect, 0, 1791 in_order); 1792 1793 virtio_update_packet_stats(&txvq->stats, txm); 1794 } 1795 1796 txvq->stats.packets += nb_tx; 1797 1798 if (likely(nb_tx)) { 1799 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1800 virtqueue_notify(vq); 1801 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1802 } 1803 } 1804 1805 return nb_tx; 1806 } 1807 1808 uint16_t 1809 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1810 { 1811 struct virtnet_tx *txvq = tx_queue; 1812 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1813 struct virtio_hw *hw = vq->hw; 1814 uint16_t hdr_size = hw->vtnet_hdr_size; 1815 uint16_t nb_used, nb_tx = 0; 1816 1817 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1818 return nb_tx; 1819 1820 if (unlikely(nb_pkts < 1)) 1821 return nb_pkts; 1822 1823 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1824 1825 nb_used = virtqueue_nused(vq); 1826 1827 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1828 virtio_xmit_cleanup(vq, nb_used); 1829 1830 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1831 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1832 int can_push = 0, use_indirect = 0, slots, need; 1833 1834 /* optimize ring usage */ 1835 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1836 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1837 rte_mbuf_refcnt_read(txm) == 1 && 1838 RTE_MBUF_DIRECT(txm) && 1839 txm->nb_segs == 1 && 1840 rte_pktmbuf_headroom(txm) >= hdr_size && 1841 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1842 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) 1843 can_push = 1; 1844 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1845 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1846 use_indirect = 1; 1847 1848 /* How many main ring entries are needed to this Tx? 1849 * any_layout => number of segments 1850 * indirect => 1 1851 * default => number of segments + 1 1852 */ 1853 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1854 need = slots - vq->vq_free_cnt; 1855 1856 /* Positive value indicates it need free vring descriptors */ 1857 if (unlikely(need > 0)) { 1858 nb_used = virtqueue_nused(vq); 1859 1860 need = RTE_MIN(need, (int)nb_used); 1861 1862 virtio_xmit_cleanup(vq, need); 1863 need = slots - vq->vq_free_cnt; 1864 if (unlikely(need > 0)) { 1865 PMD_TX_LOG(ERR, 1866 "No free tx descriptors to transmit"); 1867 break; 1868 } 1869 } 1870 1871 /* Enqueue Packet buffers */ 1872 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 1873 can_push, 0); 1874 1875 virtio_update_packet_stats(&txvq->stats, txm); 1876 } 1877 1878 txvq->stats.packets += nb_tx; 1879 1880 if (likely(nb_tx)) { 1881 vq_update_avail_idx(vq); 1882 1883 if (unlikely(virtqueue_kick_prepare(vq))) { 1884 virtqueue_notify(vq); 1885 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1886 } 1887 } 1888 1889 return nb_tx; 1890 } 1891 1892 static __rte_always_inline int 1893 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 1894 { 1895 uint16_t nb_used, nb_clean, nb_descs; 1896 1897 nb_descs = vq->vq_free_cnt + need; 1898 nb_used = virtqueue_nused(vq); 1899 nb_clean = RTE_MIN(need, (int)nb_used); 1900 1901 virtio_xmit_cleanup_inorder(vq, nb_clean); 1902 1903 return nb_descs - vq->vq_free_cnt; 1904 } 1905 1906 uint16_t 1907 virtio_xmit_pkts_inorder(void *tx_queue, 1908 struct rte_mbuf **tx_pkts, 1909 uint16_t nb_pkts) 1910 { 1911 struct virtnet_tx *txvq = tx_queue; 1912 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1913 struct virtio_hw *hw = vq->hw; 1914 uint16_t hdr_size = hw->vtnet_hdr_size; 1915 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 1916 struct rte_mbuf *inorder_pkts[nb_pkts]; 1917 int need; 1918 1919 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1920 return nb_tx; 1921 1922 if (unlikely(nb_pkts < 1)) 1923 return nb_pkts; 1924 1925 VIRTQUEUE_DUMP(vq); 1926 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1927 nb_used = virtqueue_nused(vq); 1928 1929 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1930 virtio_xmit_cleanup_inorder(vq, nb_used); 1931 1932 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1933 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1934 int slots; 1935 1936 /* optimize ring usage */ 1937 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1938 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1939 rte_mbuf_refcnt_read(txm) == 1 && 1940 RTE_MBUF_DIRECT(txm) && 1941 txm->nb_segs == 1 && 1942 rte_pktmbuf_headroom(txm) >= hdr_size && 1943 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1944 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) { 1945 inorder_pkts[nb_inorder_pkts] = txm; 1946 nb_inorder_pkts++; 1947 1948 continue; 1949 } 1950 1951 if (nb_inorder_pkts) { 1952 need = nb_inorder_pkts - vq->vq_free_cnt; 1953 if (unlikely(need > 0)) { 1954 need = virtio_xmit_try_cleanup_inorder(vq, 1955 need); 1956 if (unlikely(need > 0)) { 1957 PMD_TX_LOG(ERR, 1958 "No free tx descriptors to " 1959 "transmit"); 1960 break; 1961 } 1962 } 1963 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 1964 nb_inorder_pkts); 1965 nb_inorder_pkts = 0; 1966 } 1967 1968 slots = txm->nb_segs + 1; 1969 need = slots - vq->vq_free_cnt; 1970 if (unlikely(need > 0)) { 1971 need = virtio_xmit_try_cleanup_inorder(vq, slots); 1972 1973 if (unlikely(need > 0)) { 1974 PMD_TX_LOG(ERR, 1975 "No free tx descriptors to transmit"); 1976 break; 1977 } 1978 } 1979 /* Enqueue Packet buffers */ 1980 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 1981 1982 virtio_update_packet_stats(&txvq->stats, txm); 1983 } 1984 1985 /* Transmit all inorder packets */ 1986 if (nb_inorder_pkts) { 1987 need = nb_inorder_pkts - vq->vq_free_cnt; 1988 if (unlikely(need > 0)) { 1989 need = virtio_xmit_try_cleanup_inorder(vq, 1990 need); 1991 if (unlikely(need > 0)) { 1992 PMD_TX_LOG(ERR, 1993 "No free tx descriptors to transmit"); 1994 nb_inorder_pkts = vq->vq_free_cnt; 1995 nb_tx -= need; 1996 } 1997 } 1998 1999 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2000 nb_inorder_pkts); 2001 } 2002 2003 txvq->stats.packets += nb_tx; 2004 2005 if (likely(nb_tx)) { 2006 vq_update_avail_idx(vq); 2007 2008 if (unlikely(virtqueue_kick_prepare(vq))) { 2009 virtqueue_notify(vq); 2010 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2011 } 2012 } 2013 2014 VIRTQUEUE_DUMP(vq); 2015 2016 return nb_tx; 2017 } 2018 2019 __rte_weak uint16_t 2020 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, 2021 struct rte_mbuf **rx_pkts __rte_unused, 2022 uint16_t nb_pkts __rte_unused) 2023 { 2024 return 0; 2025 } 2026 2027 __rte_weak uint16_t 2028 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, 2029 struct rte_mbuf **tx_pkts __rte_unused, 2030 uint16_t nb_pkts __rte_unused) 2031 { 2032 return 0; 2033 } 2034