1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #include <stdalign.h> 6 #include <stdint.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <errno.h> 11 12 #include <rte_cycles.h> 13 #include <rte_memory.h> 14 #include <rte_branch_prediction.h> 15 #include <rte_mempool.h> 16 #include <rte_malloc.h> 17 #include <rte_mbuf.h> 18 #include <rte_ether.h> 19 #include <ethdev_driver.h> 20 #include <rte_prefetch.h> 21 #include <rte_string_fns.h> 22 #include <rte_errno.h> 23 #include <rte_byteorder.h> 24 #include <rte_net.h> 25 #include <rte_ip.h> 26 #include <rte_udp.h> 27 #include <rte_tcp.h> 28 29 #include "virtio_logs.h" 30 #include "virtio_ethdev.h" 31 #include "virtio.h" 32 #include "virtqueue.h" 33 #include "virtio_rxtx.h" 34 #include "virtio_rxtx_simple.h" 35 #include "virtio_ring.h" 36 37 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 38 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 39 #else 40 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 41 #endif 42 43 void 44 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num) 45 { 46 vq->vq_free_cnt += num; 47 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1); 48 } 49 50 void 51 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 52 { 53 struct vring_desc *dp, *dp_tail; 54 struct vq_desc_extra *dxp; 55 uint16_t desc_idx_last = desc_idx; 56 57 dp = &vq->vq_split.ring.desc[desc_idx]; 58 dxp = &vq->vq_descx[desc_idx]; 59 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 60 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 61 while (dp->flags & VRING_DESC_F_NEXT) { 62 desc_idx_last = dp->next; 63 dp = &vq->vq_split.ring.desc[dp->next]; 64 } 65 } 66 dxp->ndescs = 0; 67 68 /* 69 * We must append the existing free chain, if any, to the end of 70 * newly freed chain. If the virtqueue was completely used, then 71 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 72 */ 73 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 74 vq->vq_desc_head_idx = desc_idx; 75 } else { 76 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx]; 77 dp_tail->next = desc_idx; 78 } 79 80 vq->vq_desc_tail_idx = desc_idx_last; 81 dp->next = VQ_RING_DESC_CHAIN_END; 82 } 83 84 void 85 virtio_update_packet_stats(struct virtnet_stats *const stats, 86 const struct rte_mbuf *const mbuf) 87 { 88 uint32_t s = mbuf->pkt_len; 89 const struct rte_ether_addr *const ea = 90 rte_pktmbuf_mtod(mbuf, const struct rte_ether_addr *); 91 92 stats->bytes += s; 93 94 if (s >= 1024) 95 stats->size_bins[6 + (s > 1518)]++; 96 else if (s <= 64) 97 stats->size_bins[s >> 6]++; 98 else 99 stats->size_bins[32UL - rte_clz32(s) - 5]++; 100 101 RTE_BUILD_BUG_ON(offsetof(struct virtnet_stats, broadcast) != 102 offsetof(struct virtnet_stats, multicast) + sizeof(uint64_t)); 103 if (unlikely(rte_is_multicast_ether_addr(ea))) 104 (&stats->multicast)[rte_is_broadcast_ether_addr(ea)]++; 105 } 106 107 static inline void 108 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m) 109 { 110 VIRTIO_DUMP_PACKET(m, m->data_len); 111 112 virtio_update_packet_stats(&rxvq->stats, m); 113 } 114 115 static uint16_t 116 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq, 117 struct rte_mbuf **rx_pkts, 118 uint32_t *len, 119 uint16_t num) 120 { 121 struct rte_mbuf *cookie; 122 uint16_t used_idx; 123 uint16_t id; 124 struct vring_packed_desc *desc; 125 uint16_t i; 126 127 desc = vq->vq_packed.ring.desc; 128 129 for (i = 0; i < num; i++) { 130 used_idx = vq->vq_used_cons_idx; 131 /* desc_is_used has a load-acquire or rte_io_rmb inside 132 * and wait for used desc in virtqueue. 133 */ 134 if (!desc_is_used(&desc[used_idx], vq)) 135 return i; 136 len[i] = desc[used_idx].len; 137 id = desc[used_idx].id; 138 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; 139 if (unlikely(cookie == NULL)) { 140 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 141 vq->vq_used_cons_idx); 142 break; 143 } 144 rte_prefetch0(cookie); 145 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 146 rx_pkts[i] = cookie; 147 148 vq->vq_free_cnt++; 149 vq->vq_used_cons_idx++; 150 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 151 vq->vq_used_cons_idx -= vq->vq_nentries; 152 vq->vq_packed.used_wrap_counter ^= 1; 153 } 154 } 155 156 return i; 157 } 158 159 static uint16_t 160 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 161 uint32_t *len, uint16_t num) 162 { 163 struct vring_used_elem *uep; 164 struct rte_mbuf *cookie; 165 uint16_t used_idx, desc_idx; 166 uint16_t i; 167 168 /* Caller does the check */ 169 for (i = 0; i < num ; i++) { 170 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 171 uep = &vq->vq_split.ring.used->ring[used_idx]; 172 desc_idx = (uint16_t) uep->id; 173 len[i] = uep->len; 174 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 175 176 if (unlikely(cookie == NULL)) { 177 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 178 vq->vq_used_cons_idx); 179 break; 180 } 181 182 rte_prefetch0(cookie); 183 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 184 rx_pkts[i] = cookie; 185 vq->vq_used_cons_idx++; 186 vq_ring_free_chain(vq, desc_idx); 187 vq->vq_descx[desc_idx].cookie = NULL; 188 } 189 190 return i; 191 } 192 193 static uint16_t 194 virtqueue_dequeue_rx_inorder(struct virtqueue *vq, 195 struct rte_mbuf **rx_pkts, 196 uint32_t *len, 197 uint16_t num) 198 { 199 struct vring_used_elem *uep; 200 struct rte_mbuf *cookie; 201 uint16_t used_idx = 0; 202 uint16_t i; 203 204 if (unlikely(num == 0)) 205 return 0; 206 207 for (i = 0; i < num; i++) { 208 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 209 /* Desc idx same as used idx */ 210 uep = &vq->vq_split.ring.used->ring[used_idx]; 211 len[i] = uep->len; 212 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie; 213 214 if (unlikely(cookie == NULL)) { 215 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u", 216 vq->vq_used_cons_idx); 217 break; 218 } 219 220 rte_prefetch0(cookie); 221 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 222 rx_pkts[i] = cookie; 223 vq->vq_used_cons_idx++; 224 vq->vq_descx[used_idx].cookie = NULL; 225 } 226 227 vq_ring_free_inorder(vq, used_idx, i); 228 return i; 229 } 230 231 static inline int 232 virtqueue_enqueue_refill_inorder(struct virtqueue *vq, 233 struct rte_mbuf **cookies, 234 uint16_t num) 235 { 236 struct vq_desc_extra *dxp; 237 struct virtio_hw *hw = vq->hw; 238 struct vring_desc *start_dp; 239 uint16_t head_idx, idx, i = 0; 240 241 if (unlikely(vq->vq_free_cnt == 0)) 242 return -ENOSPC; 243 if (unlikely(vq->vq_free_cnt < num)) 244 return -EMSGSIZE; 245 246 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1); 247 start_dp = vq->vq_split.ring.desc; 248 249 while (i < num) { 250 idx = head_idx & (vq->vq_nentries - 1); 251 dxp = &vq->vq_descx[idx]; 252 dxp->cookie = (void *)cookies[i]; 253 dxp->ndescs = 1; 254 255 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) + 256 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 257 start_dp[idx].len = cookies[i]->buf_len - 258 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 259 start_dp[idx].flags = VRING_DESC_F_WRITE; 260 261 vq_update_avail_ring(vq, idx); 262 head_idx++; 263 i++; 264 } 265 266 vq->vq_desc_head_idx += num; 267 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 268 return 0; 269 } 270 271 static inline int 272 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie, 273 uint16_t num) 274 { 275 struct vq_desc_extra *dxp; 276 struct virtio_hw *hw = vq->hw; 277 struct vring_desc *start_dp = vq->vq_split.ring.desc; 278 uint16_t idx, i; 279 280 if (unlikely(vq->vq_free_cnt == 0)) 281 return -ENOSPC; 282 if (unlikely(vq->vq_free_cnt < num)) 283 return -EMSGSIZE; 284 285 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries)) 286 return -EFAULT; 287 288 for (i = 0; i < num; i++) { 289 idx = vq->vq_desc_head_idx; 290 dxp = &vq->vq_descx[idx]; 291 dxp->cookie = (void *)cookie[i]; 292 dxp->ndescs = 1; 293 294 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) + 295 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 296 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM + 297 hw->vtnet_hdr_size; 298 start_dp[idx].flags = VRING_DESC_F_WRITE; 299 vq->vq_desc_head_idx = start_dp[idx].next; 300 vq_update_avail_ring(vq, idx); 301 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) { 302 vq->vq_desc_tail_idx = vq->vq_desc_head_idx; 303 break; 304 } 305 } 306 307 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 308 309 return 0; 310 } 311 312 static inline void 313 virtqueue_refill_single_packed(struct virtqueue *vq, 314 struct vring_packed_desc *dp, 315 struct rte_mbuf *cookie) 316 { 317 uint16_t flags = vq->vq_packed.cached_flags; 318 struct virtio_hw *hw = vq->hw; 319 320 dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size; 321 dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 322 323 virtqueue_store_flags_packed(dp, flags, hw->weak_barriers); 324 325 if (++vq->vq_avail_idx >= vq->vq_nentries) { 326 vq->vq_avail_idx -= vq->vq_nentries; 327 vq->vq_packed.cached_flags ^= 328 VRING_PACKED_DESC_F_AVAIL_USED; 329 flags = vq->vq_packed.cached_flags; 330 } 331 } 332 333 static inline int 334 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq, 335 struct rte_mbuf **cookie, uint16_t num) 336 { 337 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 338 struct vq_desc_extra *dxp; 339 uint16_t idx; 340 int i; 341 342 if (unlikely(vq->vq_free_cnt == 0)) 343 return -ENOSPC; 344 if (unlikely(vq->vq_free_cnt < num)) 345 return -EMSGSIZE; 346 347 for (i = 0; i < num; i++) { 348 idx = vq->vq_avail_idx; 349 dxp = &vq->vq_descx[idx]; 350 dxp->cookie = (void *)cookie[i]; 351 dxp->ndescs = 1; 352 353 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 354 } 355 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 356 return 0; 357 } 358 359 static inline int 360 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, 361 struct rte_mbuf **cookie, uint16_t num) 362 { 363 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc; 364 struct vq_desc_extra *dxp; 365 uint16_t idx, did; 366 int i; 367 368 if (unlikely(vq->vq_free_cnt == 0)) 369 return -ENOSPC; 370 if (unlikely(vq->vq_free_cnt < num)) 371 return -EMSGSIZE; 372 373 for (i = 0; i < num; i++) { 374 idx = vq->vq_avail_idx; 375 did = start_dp[idx].id; 376 dxp = &vq->vq_descx[did]; 377 dxp->cookie = (void *)cookie[i]; 378 dxp->ndescs = 1; 379 380 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]); 381 } 382 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 383 return 0; 384 } 385 386 /* When doing TSO, the IP length is not included in the pseudo header 387 * checksum of the packet given to the PMD, but for virtio it is 388 * expected. 389 */ 390 static void 391 virtio_tso_fix_cksum(struct rte_mbuf *m) 392 { 393 /* common case: header is not fragmented */ 394 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len + 395 m->l4_len)) { 396 struct rte_ipv4_hdr *iph; 397 struct rte_tcp_hdr *th; 398 uint16_t prev_cksum, new_cksum; 399 uint32_t ip_paylen; 400 uint32_t tmp; 401 402 iph = rte_pktmbuf_mtod_offset(m, 403 struct rte_ipv4_hdr *, m->l2_len); 404 th = RTE_PTR_ADD(iph, m->l3_len); 405 406 /* 407 * Calculate IPv4 header checksum with current total length value 408 * (whatever it is) to have correct checksum after update on edits 409 * done by TSO. 410 */ 411 if ((iph->version_ihl >> 4) == 4) { 412 iph->hdr_checksum = 0; 413 iph->hdr_checksum = rte_ipv4_cksum(iph); 414 } 415 416 /* 417 * Do not use IPv4 total length and IPv6 payload length fields to get 418 * TSO payload length since it could not fit into 16 bits. 419 */ 420 ip_paylen = rte_cpu_to_be_32(rte_pktmbuf_pkt_len(m) - m->l2_len - 421 m->l3_len); 422 423 /* calculate the new phdr checksum not including ip_paylen */ 424 prev_cksum = th->cksum; 425 tmp = prev_cksum; 426 tmp += (ip_paylen & 0xffff) + (ip_paylen >> 16); 427 tmp = (tmp & 0xffff) + (tmp >> 16); 428 new_cksum = tmp; 429 430 /* replace it in the packet */ 431 th->cksum = new_cksum; 432 } 433 } 434 435 436 437 438 static inline void 439 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq, 440 struct rte_mbuf **cookies, 441 uint16_t num) 442 { 443 struct vq_desc_extra *dxp; 444 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 445 struct vring_desc *start_dp; 446 struct virtio_net_hdr *hdr; 447 uint16_t idx; 448 int16_t head_size = vq->hw->vtnet_hdr_size; 449 uint16_t i = 0; 450 451 idx = vq->vq_desc_head_idx; 452 start_dp = vq->vq_split.ring.desc; 453 454 while (i < num) { 455 idx = idx & (vq->vq_nentries - 1); 456 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 457 dxp->cookie = (void *)cookies[i]; 458 dxp->ndescs = 1; 459 virtio_update_packet_stats(&txvq->stats, cookies[i]); 460 461 hdr = rte_pktmbuf_mtod_offset(cookies[i], 462 struct virtio_net_hdr *, -head_size); 463 464 /* if offload disabled, hdr is not zeroed yet, do it now */ 465 if (!vq->hw->has_tx_offload) 466 virtqueue_clear_net_hdr(hdr); 467 else 468 virtqueue_xmit_offload(hdr, cookies[i]); 469 470 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size; 471 start_dp[idx].len = cookies[i]->data_len + head_size; 472 start_dp[idx].flags = 0; 473 474 475 vq_update_avail_ring(vq, idx); 476 477 idx++; 478 i++; 479 }; 480 481 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num); 482 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1); 483 } 484 485 static inline void 486 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq, 487 struct rte_mbuf *cookie, 488 int in_order) 489 { 490 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 491 struct vring_packed_desc *dp; 492 struct vq_desc_extra *dxp; 493 uint16_t idx, id, flags; 494 int16_t head_size = vq->hw->vtnet_hdr_size; 495 struct virtio_net_hdr *hdr; 496 497 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 498 idx = vq->vq_avail_idx; 499 dp = &vq->vq_packed.ring.desc[idx]; 500 501 dxp = &vq->vq_descx[id]; 502 dxp->ndescs = 1; 503 dxp->cookie = cookie; 504 505 flags = vq->vq_packed.cached_flags; 506 507 /* prepend cannot fail, checked by caller */ 508 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 509 -head_size); 510 511 /* if offload disabled, hdr is not zeroed yet, do it now */ 512 if (!vq->hw->has_tx_offload) 513 virtqueue_clear_net_hdr(hdr); 514 else 515 virtqueue_xmit_offload(hdr, cookie); 516 517 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size; 518 dp->len = cookie->data_len + head_size; 519 dp->id = id; 520 521 if (++vq->vq_avail_idx >= vq->vq_nentries) { 522 vq->vq_avail_idx -= vq->vq_nentries; 523 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 524 } 525 526 vq->vq_free_cnt--; 527 528 if (!in_order) { 529 vq->vq_desc_head_idx = dxp->next; 530 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 531 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 532 } 533 534 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers); 535 } 536 537 static inline void 538 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 539 uint16_t needed, int use_indirect, int can_push, 540 int in_order) 541 { 542 struct virtio_tx_region *txr = txvq->hdr_mz->addr; 543 struct vq_desc_extra *dxp; 544 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 545 struct vring_desc *start_dp; 546 uint16_t seg_num = cookie->nb_segs; 547 uint16_t head_idx, idx; 548 int16_t head_size = vq->hw->vtnet_hdr_size; 549 bool prepend_header = false; 550 struct virtio_net_hdr *hdr; 551 552 head_idx = vq->vq_desc_head_idx; 553 idx = head_idx; 554 if (in_order) 555 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)]; 556 else 557 dxp = &vq->vq_descx[idx]; 558 dxp->cookie = (void *)cookie; 559 dxp->ndescs = needed; 560 561 start_dp = vq->vq_split.ring.desc; 562 563 if (can_push) { 564 /* prepend cannot fail, checked by caller */ 565 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 566 -head_size); 567 prepend_header = true; 568 569 /* if offload disabled, it is not zeroed below, do it now */ 570 if (!vq->hw->has_tx_offload) 571 virtqueue_clear_net_hdr(hdr); 572 } else if (use_indirect) { 573 /* setup tx ring slot to point to indirect 574 * descriptor list stored in reserved region. 575 * 576 * the first slot in indirect ring is already preset 577 * to point to the header in reserved region 578 */ 579 start_dp[idx].addr = txvq->hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_indir, txr); 580 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc); 581 start_dp[idx].flags = VRING_DESC_F_INDIRECT; 582 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 583 584 /* loop below will fill in rest of the indirect elements */ 585 start_dp = txr[idx].tx_indir; 586 idx = 1; 587 } else { 588 /* setup first tx ring slot to point to header 589 * stored in reserved region. 590 */ 591 start_dp[idx].addr = txvq->hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 592 start_dp[idx].len = vq->hw->vtnet_hdr_size; 593 start_dp[idx].flags = VRING_DESC_F_NEXT; 594 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 595 596 idx = start_dp[idx].next; 597 } 598 599 if (vq->hw->has_tx_offload) 600 virtqueue_xmit_offload(hdr, cookie); 601 602 do { 603 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 604 start_dp[idx].len = cookie->data_len; 605 if (prepend_header) { 606 start_dp[idx].addr -= head_size; 607 start_dp[idx].len += head_size; 608 prepend_header = false; 609 } 610 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; 611 idx = start_dp[idx].next; 612 } while ((cookie = cookie->next) != NULL); 613 614 if (use_indirect) 615 idx = vq->vq_split.ring.desc[head_idx].next; 616 617 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 618 619 vq->vq_desc_head_idx = idx; 620 vq_update_avail_ring(vq, head_idx); 621 622 if (!in_order) { 623 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 624 vq->vq_desc_tail_idx = idx; 625 } 626 } 627 628 void 629 virtio_dev_cq_start(struct rte_eth_dev *dev) 630 { 631 struct virtio_hw *hw = dev->data->dev_private; 632 633 if (hw->cvq) { 634 rte_spinlock_init(&hw->cvq->lock); 635 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq)); 636 } 637 } 638 639 int 640 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 641 uint16_t queue_idx, 642 uint16_t nb_desc, 643 unsigned int socket_id __rte_unused, 644 const struct rte_eth_rxconf *rx_conf, 645 struct rte_mempool *mp) 646 { 647 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 648 struct virtio_hw *hw = dev->data->dev_private; 649 struct virtqueue *vq = hw->vqs[vq_idx]; 650 struct virtnet_rx *rxvq; 651 uint16_t rx_free_thresh; 652 uint16_t buf_size; 653 const char *error; 654 655 PMD_INIT_FUNC_TRACE(); 656 657 if (rx_conf->rx_deferred_start) { 658 PMD_INIT_LOG(ERR, "Rx deferred start is not supported"); 659 return -EINVAL; 660 } 661 662 buf_size = virtio_rx_mem_pool_buf_size(mp); 663 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size, 664 hw->rx_ol_scatter, &error)) { 665 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s", 666 queue_idx, error); 667 return -EINVAL; 668 } 669 670 rx_free_thresh = rx_conf->rx_free_thresh; 671 if (rx_free_thresh == 0) 672 rx_free_thresh = 673 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH); 674 675 if (rx_free_thresh & 0x3) { 676 PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four." 677 " (rx_free_thresh=%u port=%u queue=%u)", 678 rx_free_thresh, dev->data->port_id, queue_idx); 679 return -EINVAL; 680 } 681 682 if (rx_free_thresh >= vq->vq_nentries) { 683 PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the " 684 "number of RX entries (%u)." 685 " (rx_free_thresh=%u port=%u queue=%u)", 686 vq->vq_nentries, 687 rx_free_thresh, dev->data->port_id, queue_idx); 688 return -EINVAL; 689 } 690 vq->vq_free_thresh = rx_free_thresh; 691 692 /* 693 * For split ring vectorized path descriptors number must be 694 * equal to the ring size. 695 */ 696 if (nb_desc > vq->vq_nentries || 697 (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) { 698 nb_desc = vq->vq_nentries; 699 } 700 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 701 702 rxvq = &vq->rxq; 703 rxvq->mpool = mp; 704 dev->data->rx_queues[queue_idx] = rxvq; 705 706 return 0; 707 } 708 709 int 710 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx) 711 { 712 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 713 struct virtio_hw *hw = dev->data->dev_private; 714 struct virtqueue *vq = hw->vqs[vq_idx]; 715 struct virtnet_rx *rxvq = &vq->rxq; 716 struct rte_mbuf *m; 717 uint16_t desc_idx; 718 int error, nbufs, i; 719 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 720 721 PMD_INIT_FUNC_TRACE(); 722 723 /* Allocate blank mbufs for the each rx descriptor */ 724 nbufs = 0; 725 726 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 727 for (desc_idx = 0; desc_idx < vq->vq_nentries; 728 desc_idx++) { 729 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx; 730 vq->vq_split.ring.desc[desc_idx].flags = 731 VRING_DESC_F_WRITE; 732 } 733 734 virtio_rxq_vec_setup(rxvq); 735 } 736 737 if (hw->use_vec_rx) { 738 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf)); 739 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++) 740 vq->rxq.sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf; 741 } 742 743 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) { 744 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 745 virtio_rxq_rearm_vec(rxvq); 746 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH; 747 } 748 } else if (!virtio_with_packed_queue(vq->hw) && in_order) { 749 if ((!virtqueue_full(vq))) { 750 uint16_t free_cnt = vq->vq_free_cnt; 751 struct rte_mbuf *pkts[free_cnt]; 752 753 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts, 754 free_cnt)) { 755 error = virtqueue_enqueue_refill_inorder(vq, 756 pkts, 757 free_cnt); 758 if (unlikely(error)) { 759 for (i = 0; i < free_cnt; i++) 760 rte_pktmbuf_free(pkts[i]); 761 } else { 762 nbufs += free_cnt; 763 } 764 } 765 766 vq_update_avail_idx(vq); 767 } 768 } else { 769 while (!virtqueue_full(vq)) { 770 m = rte_mbuf_raw_alloc(rxvq->mpool); 771 if (m == NULL) 772 break; 773 774 /* Enqueue allocated buffers */ 775 if (virtio_with_packed_queue(vq->hw)) 776 error = virtqueue_enqueue_recv_refill_packed_init(vq, 777 &m, 1); 778 else 779 error = virtqueue_enqueue_recv_refill(vq, 780 &m, 1); 781 if (error) { 782 rte_pktmbuf_free(m); 783 break; 784 } 785 nbufs++; 786 } 787 788 if (!virtio_with_packed_queue(vq->hw)) 789 vq_update_avail_idx(vq); 790 } 791 792 PMD_INIT_LOG(DEBUG, "Allocated %d bufs (port=%u queue=%u)", nbufs, 793 dev->data->port_id, queue_idx); 794 795 VIRTQUEUE_DUMP(vq); 796 797 return 0; 798 } 799 800 /* 801 * struct rte_eth_dev *dev: Used to update dev 802 * uint16_t nb_desc: Defaults to values read from config space 803 * unsigned int socket_id: Used to allocate memzone 804 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 805 * uint16_t queue_idx: Just used as an index in dev txq list 806 */ 807 int 808 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 809 uint16_t queue_idx, 810 uint16_t nb_desc, 811 unsigned int socket_id __rte_unused, 812 const struct rte_eth_txconf *tx_conf) 813 { 814 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 815 struct virtio_hw *hw = dev->data->dev_private; 816 struct virtqueue *vq = hw->vqs[vq_idx]; 817 struct virtnet_tx *txvq; 818 uint16_t tx_free_thresh; 819 820 PMD_INIT_FUNC_TRACE(); 821 822 if (tx_conf->tx_deferred_start) { 823 PMD_INIT_LOG(ERR, "Tx deferred start is not supported"); 824 return -EINVAL; 825 } 826 827 if (nb_desc == 0 || nb_desc > vq->vq_nentries) 828 nb_desc = vq->vq_nentries; 829 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc); 830 831 txvq = &vq->txq; 832 833 tx_free_thresh = tx_conf->tx_free_thresh; 834 if (tx_free_thresh == 0) 835 tx_free_thresh = 836 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 837 838 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 839 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the " 840 "number of TX entries minus 3 (%u)." 841 " (tx_free_thresh=%u port=%u queue=%u)", 842 vq->vq_nentries - 3, 843 tx_free_thresh, dev->data->port_id, queue_idx); 844 return -EINVAL; 845 } 846 847 vq->vq_free_thresh = tx_free_thresh; 848 849 dev->data->tx_queues[queue_idx] = txvq; 850 return 0; 851 } 852 853 int 854 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev, 855 uint16_t queue_idx) 856 { 857 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 858 struct virtio_hw *hw = dev->data->dev_private; 859 struct virtqueue *vq = hw->vqs[vq_idx]; 860 861 PMD_INIT_FUNC_TRACE(); 862 863 if (!virtio_with_packed_queue(hw)) { 864 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 865 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0; 866 } 867 868 VIRTQUEUE_DUMP(vq); 869 870 return 0; 871 } 872 873 static inline void 874 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 875 { 876 int error; 877 /* 878 * Requeue the discarded mbuf. This should always be 879 * successful since it was just dequeued. 880 */ 881 if (virtio_with_packed_queue(vq->hw)) 882 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1); 883 else 884 error = virtqueue_enqueue_recv_refill(vq, &m, 1); 885 886 if (unlikely(error)) { 887 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 888 rte_pktmbuf_free(m); 889 } 890 } 891 892 static inline void 893 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m) 894 { 895 int error; 896 897 error = virtqueue_enqueue_refill_inorder(vq, &m, 1); 898 if (unlikely(error)) { 899 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf"); 900 rte_pktmbuf_free(m); 901 } 902 } 903 904 /* Optionally fill offload information in structure */ 905 static inline int 906 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr) 907 { 908 struct rte_net_hdr_lens hdr_lens; 909 uint32_t hdrlen, ptype; 910 int l4_supported = 0; 911 912 /* nothing to do */ 913 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 914 return 0; 915 916 m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN; 917 918 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 919 m->packet_type = ptype; 920 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP || 921 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP || 922 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP) 923 l4_supported = 1; 924 925 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 926 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len; 927 if (hdr->csum_start <= hdrlen && l4_supported) { 928 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_NONE; 929 } else { 930 /* Unknown proto or tunnel, do sw cksum. We can assume 931 * the cksum field is in the first segment since the 932 * buffers we provided to the host are large enough. 933 * In case of SCTP, this will be wrong since it's a CRC 934 * but there's nothing we can do. 935 */ 936 uint16_t csum = 0, off; 937 938 if (rte_raw_cksum_mbuf(m, hdr->csum_start, 939 rte_pktmbuf_pkt_len(m) - hdr->csum_start, 940 &csum) < 0) 941 return -EINVAL; 942 if (likely(csum != 0xffff)) 943 csum = ~csum; 944 off = hdr->csum_offset + hdr->csum_start; 945 if (rte_pktmbuf_data_len(m) >= off + 1) 946 *rte_pktmbuf_mtod_offset(m, uint16_t *, 947 off) = csum; 948 } 949 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) { 950 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 951 } 952 953 /* GSO request, save required information in mbuf */ 954 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 955 /* Check unsupported modes */ 956 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) || 957 (hdr->gso_size == 0)) { 958 return -EINVAL; 959 } 960 961 /* Update mss lengths in mbuf */ 962 m->tso_segsz = hdr->gso_size; 963 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 964 case VIRTIO_NET_HDR_GSO_TCPV4: 965 case VIRTIO_NET_HDR_GSO_TCPV6: 966 m->ol_flags |= RTE_MBUF_F_RX_LRO | 967 RTE_MBUF_F_RX_L4_CKSUM_NONE; 968 break; 969 default: 970 return -EINVAL; 971 } 972 } 973 974 return 0; 975 } 976 977 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 978 uint16_t 979 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 980 { 981 struct virtnet_rx *rxvq = rx_queue; 982 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 983 struct virtio_hw *hw = vq->hw; 984 struct rte_mbuf *rxm; 985 uint16_t nb_used, num, nb_rx; 986 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 987 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 988 int error; 989 uint32_t i, nb_enqueued; 990 uint32_t hdr_size; 991 struct virtio_net_hdr *hdr; 992 993 nb_rx = 0; 994 if (unlikely(hw->started == 0)) 995 return nb_rx; 996 997 nb_used = virtqueue_nused(vq); 998 999 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1000 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1001 num = VIRTIO_MBUF_BURST_SZ; 1002 if (likely(num > DESC_PER_CACHELINE)) 1003 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1004 1005 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1006 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 1007 1008 nb_enqueued = 0; 1009 hdr_size = hw->vtnet_hdr_size; 1010 1011 for (i = 0; i < num ; i++) { 1012 rxm = rcv_pkts[i]; 1013 1014 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1015 1016 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1017 PMD_RX_LOG(ERR, "Packet drop"); 1018 nb_enqueued++; 1019 virtio_discard_rxbuf(vq, rxm); 1020 rxvq->stats.errors++; 1021 continue; 1022 } 1023 1024 rxm->port = hw->port_id; 1025 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1026 rxm->ol_flags = 0; 1027 rxm->vlan_tci = 0; 1028 1029 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1030 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1031 1032 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1033 RTE_PKTMBUF_HEADROOM - hdr_size); 1034 1035 if (hw->vlan_strip) 1036 rte_vlan_strip(rxm); 1037 1038 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1039 virtio_discard_rxbuf(vq, rxm); 1040 rxvq->stats.errors++; 1041 continue; 1042 } 1043 1044 virtio_rx_stats_updated(rxvq, rxm); 1045 1046 rx_pkts[nb_rx++] = rxm; 1047 } 1048 1049 rxvq->stats.packets += nb_rx; 1050 1051 /* Allocate new mbuf for the used descriptor */ 1052 if (likely(!virtqueue_full(vq))) { 1053 uint16_t free_cnt = vq->vq_free_cnt; 1054 struct rte_mbuf *new_pkts[free_cnt]; 1055 1056 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1057 free_cnt) == 0)) { 1058 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1059 free_cnt); 1060 if (unlikely(error)) { 1061 for (i = 0; i < free_cnt; i++) 1062 rte_pktmbuf_free(new_pkts[i]); 1063 } 1064 nb_enqueued += free_cnt; 1065 } else { 1066 struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id]; 1067 dev->data->rx_mbuf_alloc_failed += free_cnt; 1068 } 1069 } 1070 1071 if (likely(nb_enqueued)) { 1072 vq_update_avail_idx(vq); 1073 1074 if (unlikely(virtqueue_kick_prepare(vq))) { 1075 virtqueue_notify(vq); 1076 PMD_RX_LOG(DEBUG, "Notified"); 1077 } 1078 } 1079 1080 return nb_rx; 1081 } 1082 1083 uint16_t 1084 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, 1085 uint16_t nb_pkts) 1086 { 1087 struct virtnet_rx *rxvq = rx_queue; 1088 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1089 struct virtio_hw *hw = vq->hw; 1090 struct rte_mbuf *rxm; 1091 uint16_t num, nb_rx; 1092 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1093 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1094 int error; 1095 uint32_t i, nb_enqueued; 1096 uint32_t hdr_size; 1097 struct virtio_net_hdr *hdr; 1098 1099 nb_rx = 0; 1100 if (unlikely(hw->started == 0)) 1101 return nb_rx; 1102 1103 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 1104 if (likely(num > DESC_PER_CACHELINE)) 1105 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1106 1107 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1108 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1109 1110 nb_enqueued = 0; 1111 hdr_size = hw->vtnet_hdr_size; 1112 1113 for (i = 0; i < num; i++) { 1114 rxm = rcv_pkts[i]; 1115 1116 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1117 1118 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1119 PMD_RX_LOG(ERR, "Packet drop"); 1120 nb_enqueued++; 1121 virtio_discard_rxbuf(vq, rxm); 1122 rxvq->stats.errors++; 1123 continue; 1124 } 1125 1126 rxm->port = hw->port_id; 1127 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1128 rxm->ol_flags = 0; 1129 rxm->vlan_tci = 0; 1130 1131 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1132 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1133 1134 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr + 1135 RTE_PKTMBUF_HEADROOM - hdr_size); 1136 1137 if (hw->vlan_strip) 1138 rte_vlan_strip(rxm); 1139 1140 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) { 1141 virtio_discard_rxbuf(vq, rxm); 1142 rxvq->stats.errors++; 1143 continue; 1144 } 1145 1146 virtio_rx_stats_updated(rxvq, rxm); 1147 1148 rx_pkts[nb_rx++] = rxm; 1149 } 1150 1151 rxvq->stats.packets += nb_rx; 1152 1153 /* Allocate new mbuf for the used descriptor */ 1154 if (likely(!virtqueue_full(vq))) { 1155 uint16_t free_cnt = vq->vq_free_cnt; 1156 struct rte_mbuf *new_pkts[free_cnt]; 1157 1158 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 1159 free_cnt) == 0)) { 1160 error = virtqueue_enqueue_recv_refill_packed(vq, 1161 new_pkts, free_cnt); 1162 if (unlikely(error)) { 1163 for (i = 0; i < free_cnt; i++) 1164 rte_pktmbuf_free(new_pkts[i]); 1165 } 1166 nb_enqueued += free_cnt; 1167 } else { 1168 struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id]; 1169 dev->data->rx_mbuf_alloc_failed += free_cnt; 1170 } 1171 } 1172 1173 if (likely(nb_enqueued)) { 1174 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1175 virtqueue_notify(vq); 1176 PMD_RX_LOG(DEBUG, "Notified"); 1177 } 1178 } 1179 1180 return nb_rx; 1181 } 1182 1183 1184 uint16_t 1185 virtio_recv_pkts_inorder(void *rx_queue, 1186 struct rte_mbuf **rx_pkts, 1187 uint16_t nb_pkts) 1188 { 1189 struct virtnet_rx *rxvq = rx_queue; 1190 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1191 struct virtio_hw *hw = vq->hw; 1192 struct rte_mbuf *rxm; 1193 struct rte_mbuf *prev = NULL; 1194 uint16_t nb_used, num, nb_rx; 1195 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1196 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1197 int error; 1198 uint32_t nb_enqueued; 1199 uint32_t seg_num; 1200 uint32_t seg_res; 1201 uint32_t hdr_size; 1202 int32_t i; 1203 1204 nb_rx = 0; 1205 if (unlikely(hw->started == 0)) 1206 return nb_rx; 1207 1208 nb_used = virtqueue_nused(vq); 1209 nb_used = RTE_MIN(nb_used, nb_pkts); 1210 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ); 1211 1212 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1213 1214 nb_enqueued = 0; 1215 seg_num = 1; 1216 seg_res = 0; 1217 hdr_size = hw->vtnet_hdr_size; 1218 1219 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used); 1220 1221 for (i = 0; i < num; i++) { 1222 struct virtio_net_hdr_mrg_rxbuf *header; 1223 1224 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1225 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1226 1227 rxm = rcv_pkts[i]; 1228 1229 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1230 PMD_RX_LOG(ERR, "Packet drop"); 1231 nb_enqueued++; 1232 virtio_discard_rxbuf_inorder(vq, rxm); 1233 rxvq->stats.errors++; 1234 continue; 1235 } 1236 1237 header = (struct virtio_net_hdr_mrg_rxbuf *) 1238 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1239 - hdr_size); 1240 1241 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1242 seg_num = header->num_buffers; 1243 if (seg_num == 0) 1244 seg_num = 1; 1245 } else { 1246 seg_num = 1; 1247 } 1248 1249 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1250 rxm->nb_segs = seg_num; 1251 rxm->ol_flags = 0; 1252 rxm->vlan_tci = 0; 1253 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1254 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1255 1256 rxm->port = hw->port_id; 1257 1258 rx_pkts[nb_rx] = rxm; 1259 prev = rxm; 1260 1261 if (vq->hw->has_rx_offload && 1262 virtio_rx_offload(rxm, &header->hdr) < 0) { 1263 virtio_discard_rxbuf_inorder(vq, rxm); 1264 rxvq->stats.errors++; 1265 continue; 1266 } 1267 1268 if (hw->vlan_strip) 1269 rte_vlan_strip(rx_pkts[nb_rx]); 1270 1271 seg_res = seg_num - 1; 1272 1273 /* Merge remaining segments */ 1274 while (seg_res != 0 && i < (num - 1)) { 1275 i++; 1276 1277 rxm = rcv_pkts[i]; 1278 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1279 rxm->pkt_len = (uint32_t)(len[i]); 1280 rxm->data_len = (uint16_t)(len[i]); 1281 1282 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1283 1284 prev->next = rxm; 1285 prev = rxm; 1286 seg_res -= 1; 1287 } 1288 1289 if (!seg_res) { 1290 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1291 nb_rx++; 1292 } 1293 } 1294 1295 /* Last packet still need merge segments */ 1296 while (seg_res != 0) { 1297 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1298 VIRTIO_MBUF_BURST_SZ); 1299 1300 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1301 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, 1302 rcv_cnt); 1303 uint16_t extra_idx = 0; 1304 1305 rcv_cnt = num; 1306 while (extra_idx < rcv_cnt) { 1307 rxm = rcv_pkts[extra_idx]; 1308 rxm->data_off = 1309 RTE_PKTMBUF_HEADROOM - hdr_size; 1310 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1311 rxm->data_len = (uint16_t)(len[extra_idx]); 1312 prev->next = rxm; 1313 prev = rxm; 1314 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1315 extra_idx += 1; 1316 }; 1317 seg_res -= rcv_cnt; 1318 1319 if (!seg_res) { 1320 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1321 nb_rx++; 1322 } 1323 } else { 1324 PMD_RX_LOG(ERR, 1325 "No enough segments for packet."); 1326 rte_pktmbuf_free(rx_pkts[nb_rx]); 1327 rxvq->stats.errors++; 1328 break; 1329 } 1330 } 1331 1332 rxvq->stats.packets += nb_rx; 1333 1334 /* Allocate new mbuf for the used descriptor */ 1335 1336 if (likely(!virtqueue_full(vq))) { 1337 /* free_cnt may include mrg descs */ 1338 uint16_t free_cnt = vq->vq_free_cnt; 1339 struct rte_mbuf *new_pkts[free_cnt]; 1340 1341 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1342 error = virtqueue_enqueue_refill_inorder(vq, new_pkts, 1343 free_cnt); 1344 if (unlikely(error)) { 1345 for (i = 0; i < free_cnt; i++) 1346 rte_pktmbuf_free(new_pkts[i]); 1347 } 1348 nb_enqueued += free_cnt; 1349 } else { 1350 struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id]; 1351 dev->data->rx_mbuf_alloc_failed += free_cnt; 1352 } 1353 } 1354 1355 if (likely(nb_enqueued)) { 1356 vq_update_avail_idx(vq); 1357 1358 if (unlikely(virtqueue_kick_prepare(vq))) { 1359 virtqueue_notify(vq); 1360 PMD_RX_LOG(DEBUG, "Notified"); 1361 } 1362 } 1363 1364 return nb_rx; 1365 } 1366 1367 uint16_t 1368 virtio_recv_mergeable_pkts(void *rx_queue, 1369 struct rte_mbuf **rx_pkts, 1370 uint16_t nb_pkts) 1371 { 1372 struct virtnet_rx *rxvq = rx_queue; 1373 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1374 struct virtio_hw *hw = vq->hw; 1375 struct rte_mbuf *rxm; 1376 struct rte_mbuf *prev = NULL; 1377 uint16_t nb_used, num, nb_rx = 0; 1378 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1379 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1380 int error; 1381 uint32_t nb_enqueued = 0; 1382 uint32_t seg_num = 0; 1383 uint32_t seg_res = 0; 1384 uint32_t hdr_size = hw->vtnet_hdr_size; 1385 int32_t i; 1386 1387 if (unlikely(hw->started == 0)) 1388 return nb_rx; 1389 1390 nb_used = virtqueue_nused(vq); 1391 1392 PMD_RX_LOG(DEBUG, "used:%d", nb_used); 1393 1394 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts; 1395 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1396 num = VIRTIO_MBUF_BURST_SZ; 1397 if (likely(num > DESC_PER_CACHELINE)) 1398 num = num - ((vq->vq_used_cons_idx + num) % 1399 DESC_PER_CACHELINE); 1400 1401 1402 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num); 1403 1404 for (i = 0; i < num; i++) { 1405 struct virtio_net_hdr_mrg_rxbuf *header; 1406 1407 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1408 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1409 1410 rxm = rcv_pkts[i]; 1411 1412 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1413 PMD_RX_LOG(ERR, "Packet drop"); 1414 nb_enqueued++; 1415 virtio_discard_rxbuf(vq, rxm); 1416 rxvq->stats.errors++; 1417 continue; 1418 } 1419 1420 header = (struct virtio_net_hdr_mrg_rxbuf *) 1421 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM 1422 - hdr_size); 1423 seg_num = header->num_buffers; 1424 if (seg_num == 0) 1425 seg_num = 1; 1426 1427 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1428 rxm->nb_segs = seg_num; 1429 rxm->ol_flags = 0; 1430 rxm->vlan_tci = 0; 1431 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1432 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1433 1434 rxm->port = hw->port_id; 1435 1436 rx_pkts[nb_rx] = rxm; 1437 prev = rxm; 1438 1439 if (hw->has_rx_offload && 1440 virtio_rx_offload(rxm, &header->hdr) < 0) { 1441 virtio_discard_rxbuf(vq, rxm); 1442 rxvq->stats.errors++; 1443 continue; 1444 } 1445 1446 if (hw->vlan_strip) 1447 rte_vlan_strip(rx_pkts[nb_rx]); 1448 1449 seg_res = seg_num - 1; 1450 1451 /* Merge remaining segments */ 1452 while (seg_res != 0 && i < (num - 1)) { 1453 i++; 1454 1455 rxm = rcv_pkts[i]; 1456 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1457 rxm->pkt_len = (uint32_t)(len[i]); 1458 rxm->data_len = (uint16_t)(len[i]); 1459 1460 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1461 1462 prev->next = rxm; 1463 prev = rxm; 1464 seg_res -= 1; 1465 } 1466 1467 if (!seg_res) { 1468 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1469 nb_rx++; 1470 } 1471 } 1472 1473 /* Last packet still need merge segments */ 1474 while (seg_res != 0) { 1475 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1476 VIRTIO_MBUF_BURST_SZ); 1477 1478 if (likely(virtqueue_nused(vq) >= rcv_cnt)) { 1479 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, 1480 rcv_cnt); 1481 uint16_t extra_idx = 0; 1482 1483 rcv_cnt = num; 1484 while (extra_idx < rcv_cnt) { 1485 rxm = rcv_pkts[extra_idx]; 1486 rxm->data_off = 1487 RTE_PKTMBUF_HEADROOM - hdr_size; 1488 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1489 rxm->data_len = (uint16_t)(len[extra_idx]); 1490 prev->next = rxm; 1491 prev = rxm; 1492 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1493 extra_idx += 1; 1494 }; 1495 seg_res -= rcv_cnt; 1496 1497 if (!seg_res) { 1498 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1499 nb_rx++; 1500 } 1501 } else { 1502 PMD_RX_LOG(ERR, 1503 "No enough segments for packet."); 1504 rte_pktmbuf_free(rx_pkts[nb_rx]); 1505 rxvq->stats.errors++; 1506 break; 1507 } 1508 } 1509 1510 rxvq->stats.packets += nb_rx; 1511 1512 /* Allocate new mbuf for the used descriptor */ 1513 if (likely(!virtqueue_full(vq))) { 1514 /* free_cnt may include mrg descs */ 1515 uint16_t free_cnt = vq->vq_free_cnt; 1516 struct rte_mbuf *new_pkts[free_cnt]; 1517 1518 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1519 error = virtqueue_enqueue_recv_refill(vq, new_pkts, 1520 free_cnt); 1521 if (unlikely(error)) { 1522 for (i = 0; i < free_cnt; i++) 1523 rte_pktmbuf_free(new_pkts[i]); 1524 } 1525 nb_enqueued += free_cnt; 1526 } else { 1527 struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id]; 1528 dev->data->rx_mbuf_alloc_failed += free_cnt; 1529 } 1530 } 1531 1532 if (likely(nb_enqueued)) { 1533 vq_update_avail_idx(vq); 1534 1535 if (unlikely(virtqueue_kick_prepare(vq))) { 1536 virtqueue_notify(vq); 1537 PMD_RX_LOG(DEBUG, "Notified"); 1538 } 1539 } 1540 1541 return nb_rx; 1542 } 1543 1544 uint16_t 1545 virtio_recv_mergeable_pkts_packed(void *rx_queue, 1546 struct rte_mbuf **rx_pkts, 1547 uint16_t nb_pkts) 1548 { 1549 struct virtnet_rx *rxvq = rx_queue; 1550 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 1551 struct virtio_hw *hw = vq->hw; 1552 struct rte_mbuf *rxm; 1553 struct rte_mbuf *prev = NULL; 1554 uint16_t num, nb_rx = 0; 1555 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 1556 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 1557 uint32_t nb_enqueued = 0; 1558 uint32_t seg_num = 0; 1559 uint32_t seg_res = 0; 1560 uint32_t hdr_size = hw->vtnet_hdr_size; 1561 int32_t i; 1562 int error; 1563 1564 if (unlikely(hw->started == 0)) 1565 return nb_rx; 1566 1567 1568 num = nb_pkts; 1569 if (unlikely(num > VIRTIO_MBUF_BURST_SZ)) 1570 num = VIRTIO_MBUF_BURST_SZ; 1571 if (likely(num > DESC_PER_CACHELINE)) 1572 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 1573 1574 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); 1575 1576 for (i = 0; i < num; i++) { 1577 struct virtio_net_hdr_mrg_rxbuf *header; 1578 1579 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 1580 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 1581 1582 rxm = rcv_pkts[i]; 1583 1584 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) { 1585 PMD_RX_LOG(ERR, "Packet drop"); 1586 nb_enqueued++; 1587 virtio_discard_rxbuf(vq, rxm); 1588 rxvq->stats.errors++; 1589 continue; 1590 } 1591 1592 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *) 1593 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size); 1594 seg_num = header->num_buffers; 1595 1596 if (seg_num == 0) 1597 seg_num = 1; 1598 1599 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1600 rxm->nb_segs = seg_num; 1601 rxm->ol_flags = 0; 1602 rxm->vlan_tci = 0; 1603 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 1604 rxm->data_len = (uint16_t)(len[i] - hdr_size); 1605 1606 rxm->port = hw->port_id; 1607 rx_pkts[nb_rx] = rxm; 1608 prev = rxm; 1609 1610 if (hw->has_rx_offload && 1611 virtio_rx_offload(rxm, &header->hdr) < 0) { 1612 virtio_discard_rxbuf(vq, rxm); 1613 rxvq->stats.errors++; 1614 continue; 1615 } 1616 1617 if (hw->vlan_strip) 1618 rte_vlan_strip(rx_pkts[nb_rx]); 1619 1620 seg_res = seg_num - 1; 1621 1622 /* Merge remaining segments */ 1623 while (seg_res != 0 && i < (num - 1)) { 1624 i++; 1625 1626 rxm = rcv_pkts[i]; 1627 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1628 rxm->pkt_len = (uint32_t)(len[i]); 1629 rxm->data_len = (uint16_t)(len[i]); 1630 1631 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]); 1632 1633 prev->next = rxm; 1634 prev = rxm; 1635 seg_res -= 1; 1636 } 1637 1638 if (!seg_res) { 1639 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1640 nb_rx++; 1641 } 1642 } 1643 1644 /* Last packet still need merge segments */ 1645 while (seg_res != 0) { 1646 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, 1647 VIRTIO_MBUF_BURST_SZ); 1648 uint16_t extra_idx = 0; 1649 1650 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, 1651 len, rcv_cnt); 1652 if (unlikely(rcv_cnt == 0)) { 1653 PMD_RX_LOG(ERR, "No enough segments for packet."); 1654 rte_pktmbuf_free(rx_pkts[nb_rx]); 1655 rxvq->stats.errors++; 1656 break; 1657 } 1658 1659 while (extra_idx < rcv_cnt) { 1660 rxm = rcv_pkts[extra_idx]; 1661 1662 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 1663 rxm->pkt_len = (uint32_t)(len[extra_idx]); 1664 rxm->data_len = (uint16_t)(len[extra_idx]); 1665 1666 prev->next = rxm; 1667 prev = rxm; 1668 rx_pkts[nb_rx]->pkt_len += len[extra_idx]; 1669 extra_idx += 1; 1670 } 1671 seg_res -= rcv_cnt; 1672 if (!seg_res) { 1673 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]); 1674 nb_rx++; 1675 } 1676 } 1677 1678 rxvq->stats.packets += nb_rx; 1679 1680 /* Allocate new mbuf for the used descriptor */ 1681 if (likely(!virtqueue_full(vq))) { 1682 /* free_cnt may include mrg descs */ 1683 uint16_t free_cnt = vq->vq_free_cnt; 1684 struct rte_mbuf *new_pkts[free_cnt]; 1685 1686 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { 1687 error = virtqueue_enqueue_recv_refill_packed(vq, 1688 new_pkts, free_cnt); 1689 if (unlikely(error)) { 1690 for (i = 0; i < free_cnt; i++) 1691 rte_pktmbuf_free(new_pkts[i]); 1692 } 1693 nb_enqueued += free_cnt; 1694 } else { 1695 struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id]; 1696 dev->data->rx_mbuf_alloc_failed += free_cnt; 1697 } 1698 } 1699 1700 if (likely(nb_enqueued)) { 1701 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1702 virtqueue_notify(vq); 1703 PMD_RX_LOG(DEBUG, "Notified"); 1704 } 1705 } 1706 1707 return nb_rx; 1708 } 1709 1710 uint16_t 1711 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 1712 uint16_t nb_pkts) 1713 { 1714 uint16_t nb_tx; 1715 int error; 1716 1717 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1718 struct rte_mbuf *m = tx_pkts[nb_tx]; 1719 1720 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1721 error = rte_validate_tx_offload(m); 1722 if (unlikely(error)) { 1723 rte_errno = -error; 1724 break; 1725 } 1726 #endif 1727 1728 /* Do VLAN tag insertion */ 1729 if (unlikely(m->ol_flags & RTE_MBUF_F_TX_VLAN)) { 1730 error = rte_vlan_insert(&m); 1731 /* rte_vlan_insert() may change pointer 1732 * even in the case of failure 1733 */ 1734 tx_pkts[nb_tx] = m; 1735 1736 if (unlikely(error)) { 1737 rte_errno = -error; 1738 break; 1739 } 1740 } 1741 1742 error = rte_net_intel_cksum_prepare(m); 1743 if (unlikely(error)) { 1744 rte_errno = -error; 1745 break; 1746 } 1747 1748 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 1749 virtio_tso_fix_cksum(m); 1750 } 1751 1752 return nb_tx; 1753 } 1754 1755 uint16_t 1756 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, 1757 uint16_t nb_pkts) 1758 { 1759 struct virtnet_tx *txvq = tx_queue; 1760 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1761 struct virtio_hw *hw = vq->hw; 1762 uint16_t hdr_size = hw->vtnet_hdr_size; 1763 uint16_t nb_tx = 0; 1764 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER); 1765 1766 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1767 return nb_tx; 1768 1769 if (unlikely(nb_pkts < 1)) 1770 return nb_pkts; 1771 1772 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1773 1774 if (nb_pkts > vq->vq_free_cnt) 1775 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, 1776 in_order); 1777 1778 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1779 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1780 int can_push = 0, use_indirect = 0, slots, need; 1781 1782 /* optimize ring usage */ 1783 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1784 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1785 rte_mbuf_refcnt_read(txm) == 1 && 1786 RTE_MBUF_DIRECT(txm) && 1787 txm->nb_segs == 1 && 1788 rte_pktmbuf_headroom(txm) >= hdr_size && 1789 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1790 alignof(struct virtio_net_hdr_mrg_rxbuf))) 1791 can_push = 1; 1792 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1793 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1794 use_indirect = 1; 1795 /* How many main ring entries are needed to this Tx? 1796 * indirect => 1 1797 * any_layout => number of segments 1798 * default => number of segments + 1 1799 */ 1800 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1801 need = slots - vq->vq_free_cnt; 1802 1803 /* Positive value indicates it need free vring descriptors */ 1804 if (unlikely(need > 0)) { 1805 virtio_xmit_cleanup_packed(vq, need, in_order); 1806 need = slots - vq->vq_free_cnt; 1807 if (unlikely(need > 0)) { 1808 PMD_TX_LOG(ERR, 1809 "No free tx descriptors to transmit"); 1810 break; 1811 } 1812 } 1813 1814 /* Enqueue Packet buffers */ 1815 if (can_push) 1816 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order); 1817 else 1818 virtqueue_enqueue_xmit_packed(txvq, txm, slots, 1819 use_indirect, 0, 1820 in_order); 1821 1822 virtio_update_packet_stats(&txvq->stats, txm); 1823 } 1824 1825 txvq->stats.packets += nb_tx; 1826 1827 if (likely(nb_tx)) { 1828 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 1829 virtqueue_notify(vq); 1830 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1831 } 1832 } 1833 1834 return nb_tx; 1835 } 1836 1837 uint16_t 1838 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1839 { 1840 struct virtnet_tx *txvq = tx_queue; 1841 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1842 struct virtio_hw *hw = vq->hw; 1843 uint16_t hdr_size = hw->vtnet_hdr_size; 1844 uint16_t nb_used, nb_tx = 0; 1845 1846 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1847 return nb_tx; 1848 1849 if (unlikely(nb_pkts < 1)) 1850 return nb_pkts; 1851 1852 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1853 1854 nb_used = virtqueue_nused(vq); 1855 1856 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1857 virtio_xmit_cleanup(vq, nb_used); 1858 1859 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1860 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1861 int can_push = 0, use_indirect = 0, slots, need; 1862 1863 /* optimize ring usage */ 1864 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1865 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1866 rte_mbuf_refcnt_read(txm) == 1 && 1867 RTE_MBUF_DIRECT(txm) && 1868 txm->nb_segs == 1 && 1869 rte_pktmbuf_headroom(txm) >= hdr_size && 1870 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1871 alignof(struct virtio_net_hdr_mrg_rxbuf))) 1872 can_push = 1; 1873 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) && 1874 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT) 1875 use_indirect = 1; 1876 1877 /* How many main ring entries are needed to this Tx? 1878 * any_layout => number of segments 1879 * indirect => 1 1880 * default => number of segments + 1 1881 */ 1882 slots = use_indirect ? 1 : (txm->nb_segs + !can_push); 1883 need = slots - vq->vq_free_cnt; 1884 1885 /* Positive value indicates it need free vring descriptors */ 1886 if (unlikely(need > 0)) { 1887 nb_used = virtqueue_nused(vq); 1888 1889 need = RTE_MIN(need, (int)nb_used); 1890 1891 virtio_xmit_cleanup(vq, need); 1892 need = slots - vq->vq_free_cnt; 1893 if (unlikely(need > 0)) { 1894 PMD_TX_LOG(ERR, 1895 "No free tx descriptors to transmit"); 1896 break; 1897 } 1898 } 1899 1900 /* Enqueue Packet buffers */ 1901 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect, 1902 can_push, 0); 1903 1904 virtio_update_packet_stats(&txvq->stats, txm); 1905 } 1906 1907 txvq->stats.packets += nb_tx; 1908 1909 if (likely(nb_tx)) { 1910 vq_update_avail_idx(vq); 1911 1912 if (unlikely(virtqueue_kick_prepare(vq))) { 1913 virtqueue_notify(vq); 1914 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 1915 } 1916 } 1917 1918 return nb_tx; 1919 } 1920 1921 static __rte_always_inline int 1922 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need) 1923 { 1924 uint16_t nb_used, nb_clean, nb_descs; 1925 1926 nb_descs = vq->vq_free_cnt + need; 1927 nb_used = virtqueue_nused(vq); 1928 nb_clean = RTE_MIN(need, (int)nb_used); 1929 1930 virtio_xmit_cleanup_inorder(vq, nb_clean); 1931 1932 return nb_descs - vq->vq_free_cnt; 1933 } 1934 1935 uint16_t 1936 virtio_xmit_pkts_inorder(void *tx_queue, 1937 struct rte_mbuf **tx_pkts, 1938 uint16_t nb_pkts) 1939 { 1940 struct virtnet_tx *txvq = tx_queue; 1941 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 1942 struct virtio_hw *hw = vq->hw; 1943 uint16_t hdr_size = hw->vtnet_hdr_size; 1944 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0; 1945 struct rte_mbuf *inorder_pkts[nb_pkts]; 1946 int need; 1947 1948 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 1949 return nb_tx; 1950 1951 if (unlikely(nb_pkts < 1)) 1952 return nb_pkts; 1953 1954 VIRTQUEUE_DUMP(vq); 1955 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 1956 nb_used = virtqueue_nused(vq); 1957 1958 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh)) 1959 virtio_xmit_cleanup_inorder(vq, nb_used); 1960 1961 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1962 struct rte_mbuf *txm = tx_pkts[nb_tx]; 1963 int slots; 1964 1965 /* optimize ring usage */ 1966 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) || 1967 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) && 1968 rte_mbuf_refcnt_read(txm) == 1 && 1969 RTE_MBUF_DIRECT(txm) && 1970 txm->nb_segs == 1 && 1971 rte_pktmbuf_headroom(txm) >= hdr_size && 1972 rte_is_aligned(rte_pktmbuf_mtod(txm, char *), 1973 alignof(struct virtio_net_hdr_mrg_rxbuf))) { 1974 inorder_pkts[nb_inorder_pkts] = txm; 1975 nb_inorder_pkts++; 1976 1977 continue; 1978 } 1979 1980 if (nb_inorder_pkts) { 1981 need = nb_inorder_pkts - vq->vq_free_cnt; 1982 if (unlikely(need > 0)) { 1983 need = virtio_xmit_try_cleanup_inorder(vq, 1984 need); 1985 if (unlikely(need > 0)) { 1986 PMD_TX_LOG(ERR, 1987 "No free tx descriptors to " 1988 "transmit"); 1989 break; 1990 } 1991 } 1992 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 1993 nb_inorder_pkts); 1994 nb_inorder_pkts = 0; 1995 } 1996 1997 slots = txm->nb_segs + 1; 1998 need = slots - vq->vq_free_cnt; 1999 if (unlikely(need > 0)) { 2000 need = virtio_xmit_try_cleanup_inorder(vq, slots); 2001 2002 if (unlikely(need > 0)) { 2003 PMD_TX_LOG(ERR, 2004 "No free tx descriptors to transmit"); 2005 break; 2006 } 2007 } 2008 /* Enqueue Packet buffers */ 2009 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1); 2010 2011 virtio_update_packet_stats(&txvq->stats, txm); 2012 } 2013 2014 /* Transmit all inorder packets */ 2015 if (nb_inorder_pkts) { 2016 need = nb_inorder_pkts - vq->vq_free_cnt; 2017 if (unlikely(need > 0)) { 2018 need = virtio_xmit_try_cleanup_inorder(vq, 2019 need); 2020 if (unlikely(need > 0)) { 2021 PMD_TX_LOG(ERR, 2022 "No free tx descriptors to transmit"); 2023 nb_inorder_pkts = vq->vq_free_cnt; 2024 nb_tx -= need; 2025 } 2026 } 2027 2028 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts, 2029 nb_inorder_pkts); 2030 } 2031 2032 txvq->stats.packets += nb_tx; 2033 2034 if (likely(nb_tx)) { 2035 vq_update_avail_idx(vq); 2036 2037 if (unlikely(virtqueue_kick_prepare(vq))) { 2038 virtqueue_notify(vq); 2039 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 2040 } 2041 } 2042 2043 VIRTQUEUE_DUMP(vq); 2044 2045 return nb_tx; 2046 } 2047 2048 __rte_weak uint16_t 2049 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, 2050 struct rte_mbuf **rx_pkts __rte_unused, 2051 uint16_t nb_pkts __rte_unused) 2052 { 2053 return 0; 2054 } 2055 2056 __rte_weak uint16_t 2057 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, 2058 struct rte_mbuf **tx_pkts __rte_unused, 2059 uint16_t nb_pkts __rte_unused) 2060 { 2061 return 0; 2062 } 2063