1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 /* Verbs header. */ 11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 12 #ifdef PEDANTIC 13 #pragma GCC diagnostic ignored "-Wpedantic" 14 #endif 15 #include <infiniband/verbs.h> 16 #include <infiniband/mlx5dv.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 21 #include <rte_mbuf.h> 22 #include <rte_mempool.h> 23 #include <rte_prefetch.h> 24 25 #include <mlx5_prm.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_utils.h" 30 #include "mlx5_rxtx.h" 31 #include "mlx5_rxtx_vec.h" 32 #include "mlx5_autoconf.h" 33 34 #if defined RTE_ARCH_X86_64 35 #include "mlx5_rxtx_vec_sse.h" 36 #elif defined RTE_ARCH_ARM64 37 #include "mlx5_rxtx_vec_neon.h" 38 #elif defined RTE_ARCH_PPC_64 39 #include "mlx5_rxtx_vec_altivec.h" 40 #else 41 #error "This should not be compiled if SIMD instructions are not supported." 42 #endif 43 44 /** 45 * Skip error packets. 46 * 47 * @param rxq 48 * Pointer to RX queue structure. 49 * @param[out] pkts 50 * Array to store received packets. 51 * @param pkts_n 52 * Maximum number of packets in array. 53 * 54 * @return 55 * Number of packets successfully received (<= pkts_n). 56 */ 57 static uint16_t 58 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 59 uint16_t pkts_n) 60 { 61 uint16_t n = 0; 62 unsigned int i; 63 #ifdef MLX5_PMD_SOFT_COUNTERS 64 uint32_t err_bytes = 0; 65 #endif 66 67 for (i = 0; i < pkts_n; ++i) { 68 struct rte_mbuf *pkt = pkts[i]; 69 70 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 71 #ifdef MLX5_PMD_SOFT_COUNTERS 72 err_bytes += PKT_LEN(pkt); 73 #endif 74 rte_pktmbuf_free_seg(pkt); 75 } else { 76 pkts[n++] = pkt; 77 } 78 } 79 rxq->stats.idropped += (pkts_n - n); 80 #ifdef MLX5_PMD_SOFT_COUNTERS 81 /* Correct counters of errored completions. */ 82 rxq->stats.ipackets -= (pkts_n - n); 83 rxq->stats.ibytes -= err_bytes; 84 #endif 85 mlx5_rx_err_handle(rxq, 1); 86 return n; 87 } 88 89 /** 90 * DPDK callback for vectorized RX. 91 * 92 * @param dpdk_rxq 93 * Generic pointer to RX queue structure. 94 * @param[out] pkts 95 * Array to store received packets. 96 * @param pkts_n 97 * Maximum number of packets in array. 98 * 99 * @return 100 * Number of packets successfully received (<= pkts_n). 101 */ 102 uint16_t 103 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 104 { 105 struct mlx5_rxq_data *rxq = dpdk_rxq; 106 uint16_t nb_rx = 0; 107 uint16_t tn = 0; 108 uint64_t err = 0; 109 bool no_cq = false; 110 111 do { 112 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, &err, &no_cq); 113 if (unlikely(err | rxq->err_state)) 114 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 115 tn += nb_rx; 116 if (unlikely(no_cq)) 117 break; 118 } while (tn != pkts_n); 119 return tn; 120 } 121 122 /** 123 * Check a RX queue can support vectorized RX. 124 * 125 * @param rxq 126 * Pointer to RX queue. 127 * 128 * @return 129 * 1 if supported, negative errno value if not. 130 */ 131 int __rte_cold 132 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 133 { 134 struct mlx5_rxq_ctrl *ctrl = 135 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 136 137 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 138 return -ENOTSUP; 139 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 140 return -ENOTSUP; 141 if (rxq->lro) 142 return -ENOTSUP; 143 return 1; 144 } 145 146 /** 147 * Check a device can support vectorized RX. 148 * 149 * @param dev 150 * Pointer to Ethernet device. 151 * 152 * @return 153 * 1 if supported, negative errno value if not. 154 */ 155 int __rte_cold 156 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 157 { 158 struct mlx5_priv *priv = dev->data->dev_private; 159 uint32_t i; 160 161 if (!priv->config.rx_vec_en) 162 return -ENOTSUP; 163 if (mlx5_mprq_enabled(dev)) 164 return -ENOTSUP; 165 /* All the configured queues should support. */ 166 for (i = 0; i < priv->rxqs_n; ++i) { 167 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 168 169 if (!rxq) 170 continue; 171 if (mlx5_rxq_check_vec_support(rxq) < 0) 172 break; 173 } 174 if (i != priv->rxqs_n) 175 return -ENOTSUP; 176 return 1; 177 } 178