1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 /* Verbs header. */ 11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 12 #ifdef PEDANTIC 13 #pragma GCC diagnostic ignored "-Wpedantic" 14 #endif 15 #include <infiniband/verbs.h> 16 #include <infiniband/mlx5dv.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 21 #include <rte_mbuf.h> 22 #include <rte_mempool.h> 23 #include <rte_prefetch.h> 24 25 #include <mlx5_prm.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_utils.h" 30 #include "mlx5_rxtx.h" 31 #include "mlx5_rxtx_vec.h" 32 #include "mlx5_autoconf.h" 33 34 #if defined RTE_ARCH_X86_64 35 #include "mlx5_rxtx_vec_sse.h" 36 #elif defined RTE_ARCH_ARM64 37 #include "mlx5_rxtx_vec_neon.h" 38 #elif defined RTE_ARCH_PPC_64 39 #include "mlx5_rxtx_vec_altivec.h" 40 #else 41 #error "This should not be compiled if SIMD instructions are not supported." 42 #endif 43 44 /** 45 * Skip error packets. 46 * 47 * @param rxq 48 * Pointer to RX queue structure. 49 * @param[out] pkts 50 * Array to store received packets. 51 * @param pkts_n 52 * Maximum number of packets in array. 53 * 54 * @return 55 * Number of packets successfully received (<= pkts_n). 56 */ 57 static uint16_t 58 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 59 uint16_t pkts_n) 60 { 61 uint16_t n = 0; 62 unsigned int i; 63 #ifdef MLX5_PMD_SOFT_COUNTERS 64 uint32_t err_bytes = 0; 65 #endif 66 67 for (i = 0; i < pkts_n; ++i) { 68 struct rte_mbuf *pkt = pkts[i]; 69 70 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 71 #ifdef MLX5_PMD_SOFT_COUNTERS 72 err_bytes += PKT_LEN(pkt); 73 #endif 74 rte_pktmbuf_free_seg(pkt); 75 } else { 76 pkts[n++] = pkt; 77 } 78 } 79 rxq->stats.idropped += (pkts_n - n); 80 #ifdef MLX5_PMD_SOFT_COUNTERS 81 /* Correct counters of errored completions. */ 82 rxq->stats.ipackets -= (pkts_n - n); 83 rxq->stats.ibytes -= err_bytes; 84 #endif 85 mlx5_rx_err_handle(rxq, 1); 86 return n; 87 } 88 89 /** 90 * DPDK callback for vectorized RX. 91 * 92 * @param dpdk_rxq 93 * Generic pointer to RX queue structure. 94 * @param[out] pkts 95 * Array to store received packets. 96 * @param pkts_n 97 * Maximum number of packets in array. 98 * 99 * @return 100 * Number of packets successfully received (<= pkts_n). 101 */ 102 uint16_t 103 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 104 { 105 struct mlx5_rxq_data *rxq = dpdk_rxq; 106 uint16_t nb_rx; 107 uint64_t err = 0; 108 109 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); 110 if (unlikely(err | rxq->err_state)) 111 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 112 return nb_rx; 113 } 114 115 /** 116 * Check a RX queue can support vectorized RX. 117 * 118 * @param rxq 119 * Pointer to RX queue. 120 * 121 * @return 122 * 1 if supported, negative errno value if not. 123 */ 124 int __attribute__((cold)) 125 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 126 { 127 struct mlx5_rxq_ctrl *ctrl = 128 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 129 130 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 131 return -ENOTSUP; 132 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 133 return -ENOTSUP; 134 if (rxq->lro) 135 return -ENOTSUP; 136 return 1; 137 } 138 139 /** 140 * Check a device can support vectorized RX. 141 * 142 * @param dev 143 * Pointer to Ethernet device. 144 * 145 * @return 146 * 1 if supported, negative errno value if not. 147 */ 148 int __attribute__((cold)) 149 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 150 { 151 struct mlx5_priv *priv = dev->data->dev_private; 152 uint16_t i; 153 154 if (!priv->config.rx_vec_en) 155 return -ENOTSUP; 156 if (mlx5_mprq_enabled(dev)) 157 return -ENOTSUP; 158 /* All the configured queues should support. */ 159 for (i = 0; i < priv->rxqs_n; ++i) { 160 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 161 162 if (!rxq) 163 continue; 164 if (mlx5_rxq_check_vec_support(rxq) < 0) 165 break; 166 } 167 if (i != priv->rxqs_n) 168 return -ENOTSUP; 169 return 1; 170 } 171