1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdint.h> 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <string.h> 38 #include <errno.h> 39 40 #include <rte_cycles.h> 41 #include <rte_memory.h> 42 #include <rte_memzone.h> 43 #include <rte_branch_prediction.h> 44 #include <rte_mempool.h> 45 #include <rte_malloc.h> 46 #include <rte_mbuf.h> 47 #include <rte_ether.h> 48 #include <rte_ethdev.h> 49 #include <rte_prefetch.h> 50 #include <rte_string_fns.h> 51 #include <rte_errno.h> 52 #include <rte_byteorder.h> 53 54 #include "virtio_logs.h" 55 #include "virtio_ethdev.h" 56 #include "virtio_pci.h" 57 #include "virtqueue.h" 58 #include "virtio_rxtx.h" 59 60 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 61 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len) 62 #else 63 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0) 64 #endif 65 66 67 #define VIRTIO_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \ 68 ETH_TXQ_FLAGS_NOOFFLOADS) 69 70 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 71 static int use_simple_rxtx; 72 #endif 73 74 static void 75 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 76 { 77 struct vring_desc *dp, *dp_tail; 78 struct vq_desc_extra *dxp; 79 uint16_t desc_idx_last = desc_idx; 80 81 dp = &vq->vq_ring.desc[desc_idx]; 82 dxp = &vq->vq_descx[desc_idx]; 83 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); 84 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 85 while (dp->flags & VRING_DESC_F_NEXT) { 86 desc_idx_last = dp->next; 87 dp = &vq->vq_ring.desc[dp->next]; 88 } 89 } 90 dxp->ndescs = 0; 91 92 /* 93 * We must append the existing free chain, if any, to the end of 94 * newly freed chain. If the virtqueue was completely used, then 95 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 96 */ 97 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { 98 vq->vq_desc_head_idx = desc_idx; 99 } else { 100 dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx]; 101 dp_tail->next = desc_idx; 102 } 103 104 vq->vq_desc_tail_idx = desc_idx_last; 105 dp->next = VQ_RING_DESC_CHAIN_END; 106 } 107 108 static uint16_t 109 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts, 110 uint32_t *len, uint16_t num) 111 { 112 struct vring_used_elem *uep; 113 struct rte_mbuf *cookie; 114 uint16_t used_idx, desc_idx; 115 uint16_t i; 116 117 /* Caller does the check */ 118 for (i = 0; i < num ; i++) { 119 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 120 uep = &vq->vq_ring.used->ring[used_idx]; 121 desc_idx = (uint16_t) uep->id; 122 len[i] = uep->len; 123 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie; 124 125 if (unlikely(cookie == NULL)) { 126 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n", 127 vq->vq_used_cons_idx); 128 break; 129 } 130 131 rte_prefetch0(cookie); 132 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); 133 rx_pkts[i] = cookie; 134 vq->vq_used_cons_idx++; 135 vq_ring_free_chain(vq, desc_idx); 136 vq->vq_descx[desc_idx].cookie = NULL; 137 } 138 139 return i; 140 } 141 142 #ifndef DEFAULT_TX_FREE_THRESH 143 #define DEFAULT_TX_FREE_THRESH 32 144 #endif 145 146 /* Cleanup from completed transmits. */ 147 static void 148 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) 149 { 150 uint16_t i, used_idx, desc_idx; 151 for (i = 0; i < num; i++) { 152 struct vring_used_elem *uep; 153 struct vq_desc_extra *dxp; 154 155 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1)); 156 uep = &vq->vq_ring.used->ring[used_idx]; 157 158 desc_idx = (uint16_t) uep->id; 159 dxp = &vq->vq_descx[desc_idx]; 160 vq->vq_used_cons_idx++; 161 vq_ring_free_chain(vq, desc_idx); 162 163 if (dxp->cookie != NULL) { 164 rte_pktmbuf_free(dxp->cookie); 165 dxp->cookie = NULL; 166 } 167 } 168 } 169 170 171 static inline int 172 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) 173 { 174 struct vq_desc_extra *dxp; 175 struct virtio_hw *hw = vq->hw; 176 struct vring_desc *start_dp; 177 uint16_t needed = 1; 178 uint16_t head_idx, idx; 179 180 if (unlikely(vq->vq_free_cnt == 0)) 181 return -ENOSPC; 182 if (unlikely(vq->vq_free_cnt < needed)) 183 return -EMSGSIZE; 184 185 head_idx = vq->vq_desc_head_idx; 186 if (unlikely(head_idx >= vq->vq_nentries)) 187 return -EFAULT; 188 189 idx = head_idx; 190 dxp = &vq->vq_descx[idx]; 191 dxp->cookie = (void *)cookie; 192 dxp->ndescs = needed; 193 194 start_dp = vq->vq_ring.desc; 195 start_dp[idx].addr = 196 (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM 197 - hw->vtnet_hdr_size); 198 start_dp[idx].len = 199 cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; 200 start_dp[idx].flags = VRING_DESC_F_WRITE; 201 idx = start_dp[idx].next; 202 vq->vq_desc_head_idx = idx; 203 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 204 vq->vq_desc_tail_idx = idx; 205 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 206 vq_update_avail_ring(vq, head_idx); 207 208 return 0; 209 } 210 211 static int 212 virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie) 213 { 214 struct vq_desc_extra *dxp; 215 struct vring_desc *start_dp; 216 uint16_t seg_num = cookie->nb_segs; 217 uint16_t needed = 1 + seg_num; 218 uint16_t head_idx, idx; 219 size_t head_size = txvq->hw->vtnet_hdr_size; 220 221 if (unlikely(txvq->vq_free_cnt == 0)) 222 return -ENOSPC; 223 if (unlikely(txvq->vq_free_cnt < needed)) 224 return -EMSGSIZE; 225 head_idx = txvq->vq_desc_head_idx; 226 if (unlikely(head_idx >= txvq->vq_nentries)) 227 return -EFAULT; 228 229 idx = head_idx; 230 dxp = &txvq->vq_descx[idx]; 231 dxp->cookie = (void *)cookie; 232 dxp->ndescs = needed; 233 234 start_dp = txvq->vq_ring.desc; 235 start_dp[idx].addr = 236 txvq->virtio_net_hdr_mem + idx * head_size; 237 start_dp[idx].len = head_size; 238 start_dp[idx].flags = VRING_DESC_F_NEXT; 239 240 for (; ((seg_num > 0) && (cookie != NULL)); seg_num--) { 241 idx = start_dp[idx].next; 242 start_dp[idx].addr = rte_mbuf_data_dma_addr(cookie); 243 start_dp[idx].len = cookie->data_len; 244 start_dp[idx].flags = VRING_DESC_F_NEXT; 245 cookie = cookie->next; 246 } 247 248 start_dp[idx].flags &= ~VRING_DESC_F_NEXT; 249 idx = start_dp[idx].next; 250 txvq->vq_desc_head_idx = idx; 251 if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 252 txvq->vq_desc_tail_idx = idx; 253 txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed); 254 vq_update_avail_ring(txvq, head_idx); 255 256 return 0; 257 } 258 259 static inline struct rte_mbuf * 260 rte_rxmbuf_alloc(struct rte_mempool *mp) 261 { 262 struct rte_mbuf *m; 263 264 m = __rte_mbuf_raw_alloc(mp); 265 __rte_mbuf_sanity_check_raw(m, 0); 266 267 return m; 268 } 269 270 static void 271 virtio_dev_vring_start(struct virtqueue *vq, int queue_type) 272 { 273 struct rte_mbuf *m; 274 int i, nbufs, error, size = vq->vq_nentries; 275 struct vring *vr = &vq->vq_ring; 276 uint8_t *ring_mem = vq->vq_ring_virt_mem; 277 278 PMD_INIT_FUNC_TRACE(); 279 280 /* 281 * Reinitialise since virtio port might have been stopped and restarted 282 */ 283 memset(vq->vq_ring_virt_mem, 0, vq->vq_ring_size); 284 vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN); 285 vq->vq_used_cons_idx = 0; 286 vq->vq_desc_head_idx = 0; 287 vq->vq_avail_idx = 0; 288 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); 289 vq->vq_free_cnt = vq->vq_nentries; 290 memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); 291 292 /* Chain all the descriptors in the ring with an END */ 293 for (i = 0; i < size - 1; i++) 294 vr->desc[i].next = (uint16_t)(i + 1); 295 vr->desc[i].next = VQ_RING_DESC_CHAIN_END; 296 297 /* 298 * Disable device(host) interrupting guest 299 */ 300 virtqueue_disable_intr(vq); 301 302 /* Only rx virtqueue needs mbufs to be allocated at initialization */ 303 if (queue_type == VTNET_RQ) { 304 if (vq->mpool == NULL) 305 rte_exit(EXIT_FAILURE, 306 "Cannot allocate initial mbufs for rx virtqueue"); 307 308 /* Allocate blank mbufs for the each rx descriptor */ 309 nbufs = 0; 310 error = ENOSPC; 311 312 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 313 if (use_simple_rxtx) 314 for (i = 0; i < vq->vq_nentries; i++) { 315 vq->vq_ring.avail->ring[i] = i; 316 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE; 317 } 318 #endif 319 memset(&vq->fake_mbuf, 0, sizeof(vq->fake_mbuf)); 320 for (i = 0; i < RTE_PMD_VIRTIO_RX_MAX_BURST; i++) 321 vq->sw_ring[vq->vq_nentries + i] = &vq->fake_mbuf; 322 323 while (!virtqueue_full(vq)) { 324 m = rte_rxmbuf_alloc(vq->mpool); 325 if (m == NULL) 326 break; 327 328 /****************************************** 329 * Enqueue allocated buffers * 330 *******************************************/ 331 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 332 if (use_simple_rxtx) 333 error = virtqueue_enqueue_recv_refill_simple(vq, m); 334 else 335 #endif 336 error = virtqueue_enqueue_recv_refill(vq, m); 337 if (error) { 338 rte_pktmbuf_free(m); 339 break; 340 } 341 nbufs++; 342 } 343 344 vq_update_avail_idx(vq); 345 346 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs); 347 } else if (queue_type == VTNET_TQ) { 348 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 349 if (use_simple_rxtx) { 350 int mid_idx = vq->vq_nentries >> 1; 351 for (i = 0; i < mid_idx; i++) { 352 vq->vq_ring.avail->ring[i] = i + mid_idx; 353 vq->vq_ring.desc[i + mid_idx].next = i; 354 vq->vq_ring.desc[i + mid_idx].addr = 355 vq->virtio_net_hdr_mem + 356 mid_idx * vq->hw->vtnet_hdr_size; 357 vq->vq_ring.desc[i + mid_idx].len = 358 vq->hw->vtnet_hdr_size; 359 vq->vq_ring.desc[i + mid_idx].flags = 360 VRING_DESC_F_NEXT; 361 vq->vq_ring.desc[i].flags = 0; 362 } 363 for (i = mid_idx; i < vq->vq_nentries; i++) 364 vq->vq_ring.avail->ring[i] = i; 365 } 366 #endif 367 } 368 } 369 370 void 371 virtio_dev_cq_start(struct rte_eth_dev *dev) 372 { 373 struct virtio_hw *hw = dev->data->dev_private; 374 375 if (hw->cvq) { 376 virtio_dev_vring_start(hw->cvq, VTNET_CQ); 377 VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq); 378 } 379 } 380 381 void 382 virtio_dev_rxtx_start(struct rte_eth_dev *dev) 383 { 384 /* 385 * Start receive and transmit vrings 386 * - Setup vring structure for all queues 387 * - Initialize descriptor for the rx vring 388 * - Allocate blank mbufs for the each rx descriptor 389 * 390 */ 391 int i; 392 393 PMD_INIT_FUNC_TRACE(); 394 395 /* Start rx vring. */ 396 for (i = 0; i < dev->data->nb_rx_queues; i++) { 397 virtio_dev_vring_start(dev->data->rx_queues[i], VTNET_RQ); 398 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 399 } 400 401 /* Start tx vring. */ 402 for (i = 0; i < dev->data->nb_tx_queues; i++) { 403 virtio_dev_vring_start(dev->data->tx_queues[i], VTNET_TQ); 404 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 405 } 406 } 407 408 int 409 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev, 410 uint16_t queue_idx, 411 uint16_t nb_desc, 412 unsigned int socket_id, 413 __rte_unused const struct rte_eth_rxconf *rx_conf, 414 struct rte_mempool *mp) 415 { 416 uint16_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX; 417 struct virtqueue *vq; 418 int ret; 419 420 PMD_INIT_FUNC_TRACE(); 421 ret = virtio_dev_queue_setup(dev, VTNET_RQ, queue_idx, vtpci_queue_idx, 422 nb_desc, socket_id, &vq); 423 if (ret < 0) { 424 PMD_INIT_LOG(ERR, "rvq initialization failed"); 425 return ret; 426 } 427 428 /* Create mempool for rx mbuf allocation */ 429 vq->mpool = mp; 430 431 dev->data->rx_queues[queue_idx] = vq; 432 433 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 434 virtio_rxq_vec_setup(vq); 435 #endif 436 437 return 0; 438 } 439 440 void 441 virtio_dev_rx_queue_release(void *rxq) 442 { 443 virtio_dev_queue_release(rxq); 444 } 445 446 /* 447 * struct rte_eth_dev *dev: Used to update dev 448 * uint16_t nb_desc: Defaults to values read from config space 449 * unsigned int socket_id: Used to allocate memzone 450 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine 451 * uint16_t queue_idx: Just used as an index in dev txq list 452 */ 453 int 454 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, 455 uint16_t queue_idx, 456 uint16_t nb_desc, 457 unsigned int socket_id, 458 const struct rte_eth_txconf *tx_conf) 459 { 460 uint8_t vtpci_queue_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX; 461 462 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 463 struct virtio_hw *hw = dev->data->dev_private; 464 #endif 465 struct virtqueue *vq; 466 uint16_t tx_free_thresh; 467 int ret; 468 469 PMD_INIT_FUNC_TRACE(); 470 471 if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMS) 472 != ETH_TXQ_FLAGS_NOXSUMS) { 473 PMD_INIT_LOG(ERR, "TX checksum offload not supported\n"); 474 return -EINVAL; 475 } 476 477 #ifdef RTE_MACHINE_CPUFLAG_SSSE3 478 /* Use simple rx/tx func if single segment and no offloads */ 479 if ((tx_conf->txq_flags & VIRTIO_SIMPLE_FLAGS) == VIRTIO_SIMPLE_FLAGS && 480 !vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 481 PMD_INIT_LOG(INFO, "Using simple rx/tx path"); 482 dev->tx_pkt_burst = virtio_xmit_pkts_simple; 483 dev->rx_pkt_burst = virtio_recv_pkts_vec; 484 use_simple_rxtx = 1; 485 } 486 #endif 487 488 ret = virtio_dev_queue_setup(dev, VTNET_TQ, queue_idx, vtpci_queue_idx, 489 nb_desc, socket_id, &vq); 490 if (ret < 0) { 491 PMD_INIT_LOG(ERR, "rvq initialization failed"); 492 return ret; 493 } 494 495 tx_free_thresh = tx_conf->tx_free_thresh; 496 if (tx_free_thresh == 0) 497 tx_free_thresh = 498 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH); 499 500 if (tx_free_thresh >= (vq->vq_nentries - 3)) { 501 RTE_LOG(ERR, PMD, "tx_free_thresh must be less than the " 502 "number of TX entries minus 3 (%u)." 503 " (tx_free_thresh=%u port=%u queue=%u)\n", 504 vq->vq_nentries - 3, 505 tx_free_thresh, dev->data->port_id, queue_idx); 506 return -EINVAL; 507 } 508 509 vq->vq_free_thresh = tx_free_thresh; 510 511 dev->data->tx_queues[queue_idx] = vq; 512 return 0; 513 } 514 515 void 516 virtio_dev_tx_queue_release(void *txq) 517 { 518 virtio_dev_queue_release(txq); 519 } 520 521 static void 522 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m) 523 { 524 int error; 525 /* 526 * Requeue the discarded mbuf. This should always be 527 * successful since it was just dequeued. 528 */ 529 error = virtqueue_enqueue_recv_refill(vq, m); 530 if (unlikely(error)) { 531 RTE_LOG(ERR, PMD, "cannot requeue discarded mbuf"); 532 rte_pktmbuf_free(m); 533 } 534 } 535 536 static void 537 virtio_update_packet_stats(struct virtqueue *vq, struct rte_mbuf *mbuf) 538 { 539 uint32_t s = mbuf->pkt_len; 540 struct ether_addr *ea; 541 542 if (s == 64) { 543 vq->size_bins[1]++; 544 } else if (s > 64 && s < 1024) { 545 uint32_t bin; 546 547 /* count zeros, and offset into correct bin */ 548 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5; 549 vq->size_bins[bin]++; 550 } else { 551 if (s < 64) 552 vq->size_bins[0]++; 553 else if (s < 1519) 554 vq->size_bins[6]++; 555 else if (s >= 1519) 556 vq->size_bins[7]++; 557 } 558 559 ea = rte_pktmbuf_mtod(mbuf, struct ether_addr *); 560 vq->multicast += is_multicast_ether_addr(ea); 561 vq->broadcast += is_broadcast_ether_addr(ea); 562 } 563 564 #define VIRTIO_MBUF_BURST_SZ 64 565 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc)) 566 uint16_t 567 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 568 { 569 struct virtqueue *rxvq = rx_queue; 570 struct virtio_hw *hw; 571 struct rte_mbuf *rxm, *new_mbuf; 572 uint16_t nb_used, num, nb_rx; 573 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 574 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 575 int error; 576 uint32_t i, nb_enqueued; 577 uint32_t hdr_size; 578 579 nb_used = VIRTQUEUE_NUSED(rxvq); 580 581 virtio_rmb(); 582 583 num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts); 584 num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ); 585 if (likely(num > DESC_PER_CACHELINE)) 586 num = num - ((rxvq->vq_used_cons_idx + num) % DESC_PER_CACHELINE); 587 588 if (num == 0) 589 return 0; 590 591 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, num); 592 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num); 593 594 hw = rxvq->hw; 595 nb_rx = 0; 596 nb_enqueued = 0; 597 hdr_size = hw->vtnet_hdr_size; 598 599 for (i = 0; i < num ; i++) { 600 rxm = rcv_pkts[i]; 601 602 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]); 603 604 if (unlikely(len[i] < hdr_size + ETHER_HDR_LEN)) { 605 PMD_RX_LOG(ERR, "Packet drop"); 606 nb_enqueued++; 607 virtio_discard_rxbuf(rxvq, rxm); 608 rxvq->errors++; 609 continue; 610 } 611 612 rxm->port = rxvq->port_id; 613 rxm->data_off = RTE_PKTMBUF_HEADROOM; 614 rxm->ol_flags = 0; 615 rxm->vlan_tci = 0; 616 617 rxm->nb_segs = 1; 618 rxm->next = NULL; 619 rxm->pkt_len = (uint32_t)(len[i] - hdr_size); 620 rxm->data_len = (uint16_t)(len[i] - hdr_size); 621 622 if (hw->vlan_strip) 623 rte_vlan_strip(rxm); 624 625 VIRTIO_DUMP_PACKET(rxm, rxm->data_len); 626 627 rx_pkts[nb_rx++] = rxm; 628 629 rxvq->bytes += rx_pkts[nb_rx - 1]->pkt_len; 630 virtio_update_packet_stats(rxvq, rxm); 631 } 632 633 rxvq->packets += nb_rx; 634 635 /* Allocate new mbuf for the used descriptor */ 636 error = ENOSPC; 637 while (likely(!virtqueue_full(rxvq))) { 638 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool); 639 if (unlikely(new_mbuf == NULL)) { 640 struct rte_eth_dev *dev 641 = &rte_eth_devices[rxvq->port_id]; 642 dev->data->rx_mbuf_alloc_failed++; 643 break; 644 } 645 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf); 646 if (unlikely(error)) { 647 rte_pktmbuf_free(new_mbuf); 648 break; 649 } 650 nb_enqueued++; 651 } 652 653 if (likely(nb_enqueued)) { 654 vq_update_avail_idx(rxvq); 655 656 if (unlikely(virtqueue_kick_prepare(rxvq))) { 657 virtqueue_notify(rxvq); 658 PMD_RX_LOG(DEBUG, "Notified\n"); 659 } 660 } 661 662 return nb_rx; 663 } 664 665 uint16_t 666 virtio_recv_mergeable_pkts(void *rx_queue, 667 struct rte_mbuf **rx_pkts, 668 uint16_t nb_pkts) 669 { 670 struct virtqueue *rxvq = rx_queue; 671 struct virtio_hw *hw; 672 struct rte_mbuf *rxm, *new_mbuf; 673 uint16_t nb_used, num, nb_rx; 674 uint32_t len[VIRTIO_MBUF_BURST_SZ]; 675 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ]; 676 struct rte_mbuf *prev; 677 int error; 678 uint32_t i, nb_enqueued; 679 uint32_t seg_num; 680 uint16_t extra_idx; 681 uint32_t seg_res; 682 uint32_t hdr_size; 683 684 nb_used = VIRTQUEUE_NUSED(rxvq); 685 686 virtio_rmb(); 687 688 if (nb_used == 0) 689 return 0; 690 691 PMD_RX_LOG(DEBUG, "used:%d\n", nb_used); 692 693 hw = rxvq->hw; 694 nb_rx = 0; 695 i = 0; 696 nb_enqueued = 0; 697 seg_num = 0; 698 extra_idx = 0; 699 seg_res = 0; 700 hdr_size = hw->vtnet_hdr_size; 701 702 while (i < nb_used) { 703 struct virtio_net_hdr_mrg_rxbuf *header; 704 705 if (nb_rx == nb_pkts) 706 break; 707 708 num = virtqueue_dequeue_burst_rx(rxvq, rcv_pkts, len, 1); 709 if (num != 1) 710 continue; 711 712 i++; 713 714 PMD_RX_LOG(DEBUG, "dequeue:%d\n", num); 715 PMD_RX_LOG(DEBUG, "packet len:%d\n", len[0]); 716 717 rxm = rcv_pkts[0]; 718 719 if (unlikely(len[0] < hdr_size + ETHER_HDR_LEN)) { 720 PMD_RX_LOG(ERR, "Packet drop\n"); 721 nb_enqueued++; 722 virtio_discard_rxbuf(rxvq, rxm); 723 rxvq->errors++; 724 continue; 725 } 726 727 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)rxm->buf_addr + 728 RTE_PKTMBUF_HEADROOM - hdr_size); 729 seg_num = header->num_buffers; 730 731 if (seg_num == 0) 732 seg_num = 1; 733 734 rxm->data_off = RTE_PKTMBUF_HEADROOM; 735 rxm->nb_segs = seg_num; 736 rxm->next = NULL; 737 rxm->ol_flags = 0; 738 rxm->vlan_tci = 0; 739 rxm->pkt_len = (uint32_t)(len[0] - hdr_size); 740 rxm->data_len = (uint16_t)(len[0] - hdr_size); 741 742 rxm->port = rxvq->port_id; 743 rx_pkts[nb_rx] = rxm; 744 prev = rxm; 745 746 seg_res = seg_num - 1; 747 748 while (seg_res != 0) { 749 /* 750 * Get extra segments for current uncompleted packet. 751 */ 752 uint16_t rcv_cnt = 753 RTE_MIN(seg_res, RTE_DIM(rcv_pkts)); 754 if (likely(VIRTQUEUE_NUSED(rxvq) >= rcv_cnt)) { 755 uint32_t rx_num = 756 virtqueue_dequeue_burst_rx(rxvq, 757 rcv_pkts, len, rcv_cnt); 758 i += rx_num; 759 rcv_cnt = rx_num; 760 } else { 761 PMD_RX_LOG(ERR, 762 "No enough segments for packet.\n"); 763 nb_enqueued++; 764 virtio_discard_rxbuf(rxvq, rxm); 765 rxvq->errors++; 766 break; 767 } 768 769 extra_idx = 0; 770 771 while (extra_idx < rcv_cnt) { 772 rxm = rcv_pkts[extra_idx]; 773 774 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size; 775 rxm->next = NULL; 776 rxm->pkt_len = (uint32_t)(len[extra_idx]); 777 rxm->data_len = (uint16_t)(len[extra_idx]); 778 779 if (prev) 780 prev->next = rxm; 781 782 prev = rxm; 783 rx_pkts[nb_rx]->pkt_len += rxm->pkt_len; 784 extra_idx++; 785 }; 786 seg_res -= rcv_cnt; 787 } 788 789 if (hw->vlan_strip) 790 rte_vlan_strip(rx_pkts[nb_rx]); 791 792 VIRTIO_DUMP_PACKET(rx_pkts[nb_rx], 793 rx_pkts[nb_rx]->data_len); 794 795 rxvq->bytes += rx_pkts[nb_rx]->pkt_len; 796 virtio_update_packet_stats(rxvq, rx_pkts[nb_rx]); 797 nb_rx++; 798 } 799 800 rxvq->packets += nb_rx; 801 802 /* Allocate new mbuf for the used descriptor */ 803 error = ENOSPC; 804 while (likely(!virtqueue_full(rxvq))) { 805 new_mbuf = rte_rxmbuf_alloc(rxvq->mpool); 806 if (unlikely(new_mbuf == NULL)) { 807 struct rte_eth_dev *dev 808 = &rte_eth_devices[rxvq->port_id]; 809 dev->data->rx_mbuf_alloc_failed++; 810 break; 811 } 812 error = virtqueue_enqueue_recv_refill(rxvq, new_mbuf); 813 if (unlikely(error)) { 814 rte_pktmbuf_free(new_mbuf); 815 break; 816 } 817 nb_enqueued++; 818 } 819 820 if (likely(nb_enqueued)) { 821 vq_update_avail_idx(rxvq); 822 823 if (unlikely(virtqueue_kick_prepare(rxvq))) { 824 virtqueue_notify(rxvq); 825 PMD_RX_LOG(DEBUG, "Notified"); 826 } 827 } 828 829 return nb_rx; 830 } 831 832 uint16_t 833 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 834 { 835 struct virtqueue *txvq = tx_queue; 836 uint16_t nb_used, nb_tx; 837 int error; 838 839 if (unlikely(nb_pkts < 1)) 840 return nb_pkts; 841 842 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 843 nb_used = VIRTQUEUE_NUSED(txvq); 844 845 virtio_rmb(); 846 if (likely(nb_used > txvq->vq_nentries - txvq->vq_free_thresh)) 847 virtio_xmit_cleanup(txvq, nb_used); 848 849 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 850 struct rte_mbuf *txm = tx_pkts[nb_tx]; 851 /* Need one more descriptor for virtio header. */ 852 int need = txm->nb_segs - txvq->vq_free_cnt + 1; 853 854 /* Positive value indicates it need free vring descriptors */ 855 if (unlikely(need > 0)) { 856 nb_used = VIRTQUEUE_NUSED(txvq); 857 virtio_rmb(); 858 need = RTE_MIN(need, (int)nb_used); 859 860 virtio_xmit_cleanup(txvq, need); 861 need = txm->nb_segs - txvq->vq_free_cnt + 1; 862 if (unlikely(need > 0)) { 863 PMD_TX_LOG(ERR, 864 "No free tx descriptors to transmit"); 865 break; 866 } 867 } 868 869 /* Do VLAN tag insertion */ 870 if (unlikely(txm->ol_flags & PKT_TX_VLAN_PKT)) { 871 error = rte_vlan_insert(&txm); 872 if (unlikely(error)) { 873 rte_pktmbuf_free(txm); 874 continue; 875 } 876 } 877 878 /* Enqueue Packet buffers */ 879 error = virtqueue_enqueue_xmit(txvq, txm); 880 if (unlikely(error)) { 881 if (error == ENOSPC) 882 PMD_TX_LOG(ERR, "virtqueue_enqueue Free count = 0"); 883 else if (error == EMSGSIZE) 884 PMD_TX_LOG(ERR, "virtqueue_enqueue Free count < 1"); 885 else 886 PMD_TX_LOG(ERR, "virtqueue_enqueue error: %d", error); 887 break; 888 } 889 890 txvq->bytes += txm->pkt_len; 891 virtio_update_packet_stats(txvq, txm); 892 } 893 894 txvq->packets += nb_tx; 895 896 if (likely(nb_tx)) { 897 vq_update_avail_idx(txvq); 898 899 if (unlikely(virtqueue_kick_prepare(txvq))) { 900 virtqueue_notify(txvq); 901 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 902 } 903 } 904 905 return nb_tx; 906 } 907