1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2020 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_net.h> 12 13 #include "virtio_logs.h" 14 #include "virtio_ethdev.h" 15 #include "virtio_pci.h" 16 #include "virtio_rxtx_packed.h" 17 #include "virtqueue.h" 18 19 #ifdef CC_AVX512_SUPPORT 20 #include "virtio_rxtx_packed_avx.h" 21 #elif defined(RTE_ARCH_ARM) 22 #include "virtio_rxtx_packed_neon.h" 23 #endif 24 25 uint16_t 26 virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 27 uint16_t nb_pkts) 28 { 29 struct virtnet_tx *txvq = tx_queue; 30 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 31 struct virtio_hw *hw = vq->hw; 32 uint16_t nb_tx = 0; 33 uint16_t remained; 34 35 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts)) 36 return nb_tx; 37 38 if (unlikely(nb_pkts < 1)) 39 return nb_pkts; 40 41 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts); 42 43 if (vq->vq_free_cnt <= vq->vq_nentries - vq->vq_free_thresh) 44 virtio_xmit_cleanup_inorder_packed(vq, vq->vq_free_thresh); 45 46 remained = RTE_MIN(nb_pkts, vq->vq_free_cnt); 47 48 while (remained) { 49 if (remained >= PACKED_BATCH_SIZE) { 50 if (!virtqueue_enqueue_batch_packed_vec(txvq, 51 &tx_pkts[nb_tx])) { 52 nb_tx += PACKED_BATCH_SIZE; 53 remained -= PACKED_BATCH_SIZE; 54 continue; 55 } 56 } 57 if (!virtqueue_enqueue_single_packed_vec(txvq, 58 tx_pkts[nb_tx])) { 59 nb_tx++; 60 remained--; 61 continue; 62 } 63 break; 64 }; 65 66 txvq->stats.packets += nb_tx; 67 68 if (likely(nb_tx)) { 69 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 70 virtqueue_notify(vq); 71 PMD_TX_LOG(DEBUG, "Notified backend after xmit"); 72 } 73 } 74 75 return nb_tx; 76 } 77 78 uint16_t 79 virtio_recv_pkts_packed_vec(void *rx_queue, 80 struct rte_mbuf **rx_pkts, 81 uint16_t nb_pkts) 82 { 83 struct virtnet_rx *rxvq = rx_queue; 84 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 85 struct virtio_hw *hw = vq->hw; 86 uint16_t num, nb_rx = 0; 87 uint32_t nb_enqueued = 0; 88 uint16_t free_cnt = vq->vq_free_thresh; 89 90 if (unlikely(hw->started == 0)) 91 return nb_rx; 92 93 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts); 94 if (likely(num > PACKED_BATCH_SIZE)) 95 num = num - ((vq->vq_used_cons_idx + num) % PACKED_BATCH_SIZE); 96 97 while (num) { 98 if (num >= PACKED_BATCH_SIZE) { 99 if (!virtqueue_dequeue_batch_packed_vec(rxvq, 100 &rx_pkts[nb_rx])) { 101 nb_rx += PACKED_BATCH_SIZE; 102 num -= PACKED_BATCH_SIZE; 103 continue; 104 } 105 } 106 if (!virtqueue_dequeue_single_packed_vec(rxvq, 107 &rx_pkts[nb_rx])) { 108 nb_rx++; 109 num--; 110 continue; 111 } 112 break; 113 }; 114 115 PMD_RX_LOG(DEBUG, "dequeue:%d", num); 116 117 rxvq->stats.packets += nb_rx; 118 119 if (likely(vq->vq_free_cnt >= free_cnt)) { 120 struct rte_mbuf *new_pkts[free_cnt]; 121 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, 122 free_cnt) == 0)) { 123 virtio_recv_refill_packed_vec(rxvq, new_pkts, 124 free_cnt); 125 nb_enqueued += free_cnt; 126 } else { 127 struct rte_eth_dev *dev = 128 &rte_eth_devices[rxvq->port_id]; 129 dev->data->rx_mbuf_alloc_failed += free_cnt; 130 } 131 } 132 133 if (likely(nb_enqueued)) { 134 if (unlikely(virtqueue_kick_prepare_packed(vq))) { 135 virtqueue_notify(vq); 136 PMD_RX_LOG(DEBUG, "Notified"); 137 } 138 } 139 140 return nb_rx; 141 } 142