1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <assert.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdlib.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #include <infiniband/mlx5dv.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_mbuf.h> 23 #include <rte_mempool.h> 24 #include <rte_prefetch.h> 25 26 #include "mlx5.h" 27 #include "mlx5_utils.h" 28 #include "mlx5_rxtx.h" 29 #include "mlx5_rxtx_vec.h" 30 #include "mlx5_autoconf.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_prm.h" 33 34 #if defined RTE_ARCH_X86_64 35 #include "mlx5_rxtx_vec_sse.h" 36 #elif defined RTE_ARCH_ARM64 37 #include "mlx5_rxtx_vec_neon.h" 38 #else 39 #error "This should not be compiled if SIMD instructions are not supported." 40 #endif 41 42 /** 43 * Skip error packets. 44 * 45 * @param rxq 46 * Pointer to RX queue structure. 47 * @param[out] pkts 48 * Array to store received packets. 49 * @param pkts_n 50 * Maximum number of packets in array. 51 * 52 * @return 53 * Number of packets successfully received (<= pkts_n). 54 */ 55 static uint16_t 56 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 57 uint16_t pkts_n) 58 { 59 uint16_t n = 0; 60 unsigned int i; 61 #ifdef MLX5_PMD_SOFT_COUNTERS 62 uint32_t err_bytes = 0; 63 #endif 64 65 for (i = 0; i < pkts_n; ++i) { 66 struct rte_mbuf *pkt = pkts[i]; 67 68 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 69 #ifdef MLX5_PMD_SOFT_COUNTERS 70 err_bytes += PKT_LEN(pkt); 71 #endif 72 rte_pktmbuf_free_seg(pkt); 73 } else { 74 pkts[n++] = pkt; 75 } 76 } 77 rxq->stats.idropped += (pkts_n - n); 78 #ifdef MLX5_PMD_SOFT_COUNTERS 79 /* Correct counters of errored completions. */ 80 rxq->stats.ipackets -= (pkts_n - n); 81 rxq->stats.ibytes -= err_bytes; 82 #endif 83 mlx5_rx_err_handle(rxq, 1); 84 return n; 85 } 86 87 /** 88 * DPDK callback for vectorized RX. 89 * 90 * @param dpdk_rxq 91 * Generic pointer to RX queue structure. 92 * @param[out] pkts 93 * Array to store received packets. 94 * @param pkts_n 95 * Maximum number of packets in array. 96 * 97 * @return 98 * Number of packets successfully received (<= pkts_n). 99 */ 100 uint16_t 101 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 102 { 103 struct mlx5_rxq_data *rxq = dpdk_rxq; 104 uint16_t nb_rx; 105 uint64_t err = 0; 106 107 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); 108 if (unlikely(err | rxq->err_state)) 109 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 110 return nb_rx; 111 } 112 113 /** 114 * Check a RX queue can support vectorized RX. 115 * 116 * @param rxq 117 * Pointer to RX queue. 118 * 119 * @return 120 * 1 if supported, negative errno value if not. 121 */ 122 int __attribute__((cold)) 123 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 124 { 125 struct mlx5_rxq_ctrl *ctrl = 126 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 127 128 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 129 return -ENOTSUP; 130 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 131 return -ENOTSUP; 132 if (rxq->lro) 133 return -ENOTSUP; 134 return 1; 135 } 136 137 /** 138 * Check a device can support vectorized RX. 139 * 140 * @param dev 141 * Pointer to Ethernet device. 142 * 143 * @return 144 * 1 if supported, negative errno value if not. 145 */ 146 int __attribute__((cold)) 147 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 148 { 149 struct mlx5_priv *priv = dev->data->dev_private; 150 uint16_t i; 151 152 if (!priv->config.rx_vec_en) 153 return -ENOTSUP; 154 if (mlx5_mprq_enabled(dev)) 155 return -ENOTSUP; 156 /* All the configured queues should support. */ 157 for (i = 0; i < priv->rxqs_n; ++i) { 158 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 159 160 if (!rxq) 161 continue; 162 if (mlx5_rxq_check_vec_support(rxq) < 0) 163 break; 164 } 165 if (i != priv->rxqs_n) 166 return -ENOTSUP; 167 return 1; 168 } 169