1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015-2019 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 /* Verbs header. */ 11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 12 #ifdef PEDANTIC 13 #pragma GCC diagnostic ignored "-Wpedantic" 14 #endif 15 #include <infiniband/verbs.h> 16 #include <infiniband/mlx5dv.h> 17 #ifdef PEDANTIC 18 #pragma GCC diagnostic error "-Wpedantic" 19 #endif 20 21 #include <rte_mbuf.h> 22 #include <rte_mempool.h> 23 #include <rte_prefetch.h> 24 #include <rte_common.h> 25 #include <rte_branch_prediction.h> 26 #include <rte_ether.h> 27 #include <rte_cycles.h> 28 #include <rte_flow.h> 29 30 #include <mlx5_devx_cmds.h> 31 #include <mlx5_prm.h> 32 #include <mlx5_common.h> 33 34 #include "mlx5_defs.h" 35 #include "mlx5.h" 36 #include "mlx5_mr.h" 37 #include "mlx5_utils.h" 38 #include "mlx5_rxtx.h" 39 #include "mlx5_autoconf.h" 40 41 /* TX burst subroutines return codes. */ 42 enum mlx5_txcmp_code { 43 MLX5_TXCMP_CODE_EXIT = 0, 44 MLX5_TXCMP_CODE_ERROR, 45 MLX5_TXCMP_CODE_SINGLE, 46 MLX5_TXCMP_CODE_MULTI, 47 MLX5_TXCMP_CODE_TSO, 48 MLX5_TXCMP_CODE_EMPW, 49 }; 50 51 /* 52 * These defines are used to configure Tx burst routine option set 53 * supported at compile time. The not specified options are optimized out 54 * out due to if conditions can be explicitly calculated at compile time. 55 * The offloads with bigger runtime check (require more CPU cycles to 56 * skip) overhead should have the bigger index - this is needed to 57 * select the better matching routine function if no exact match and 58 * some offloads are not actually requested. 59 */ 60 #define MLX5_TXOFF_CONFIG_MULTI (1u << 0) /* Multi-segment packets.*/ 61 #define MLX5_TXOFF_CONFIG_TSO (1u << 1) /* TCP send offload supported.*/ 62 #define MLX5_TXOFF_CONFIG_SWP (1u << 2) /* Tunnels/SW Parser offloads.*/ 63 #define MLX5_TXOFF_CONFIG_CSUM (1u << 3) /* Check Sums offloaded. */ 64 #define MLX5_TXOFF_CONFIG_INLINE (1u << 4) /* Data inlining supported. */ 65 #define MLX5_TXOFF_CONFIG_VLAN (1u << 5) /* VLAN insertion supported.*/ 66 #define MLX5_TXOFF_CONFIG_METADATA (1u << 6) /* Flow metadata. */ 67 #define MLX5_TXOFF_CONFIG_EMPW (1u << 8) /* Enhanced MPW supported.*/ 68 #define MLX5_TXOFF_CONFIG_MPW (1u << 9) /* Legacy MPW supported.*/ 69 70 /* The most common offloads groups. */ 71 #define MLX5_TXOFF_CONFIG_NONE 0 72 #define MLX5_TXOFF_CONFIG_FULL (MLX5_TXOFF_CONFIG_MULTI | \ 73 MLX5_TXOFF_CONFIG_TSO | \ 74 MLX5_TXOFF_CONFIG_SWP | \ 75 MLX5_TXOFF_CONFIG_CSUM | \ 76 MLX5_TXOFF_CONFIG_INLINE | \ 77 MLX5_TXOFF_CONFIG_VLAN | \ 78 MLX5_TXOFF_CONFIG_METADATA) 79 80 #define MLX5_TXOFF_CONFIG(mask) (olx & MLX5_TXOFF_CONFIG_##mask) 81 82 #define MLX5_TXOFF_DECL(func, olx) \ 83 static uint16_t mlx5_tx_burst_##func(void *txq, \ 84 struct rte_mbuf **pkts, \ 85 uint16_t pkts_n) \ 86 { \ 87 return mlx5_tx_burst_tmpl((struct mlx5_txq_data *)txq, \ 88 pkts, pkts_n, (olx)); \ 89 } 90 91 #define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx}, 92 93 static __rte_always_inline uint32_t 94 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe); 95 96 static __rte_always_inline int 97 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 98 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe); 99 100 static __rte_always_inline uint32_t 101 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe); 102 103 static __rte_always_inline void 104 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, 105 volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res); 106 107 static __rte_always_inline void 108 mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx, 109 const unsigned int strd_n); 110 111 static int 112 mlx5_queue_state_modify(struct rte_eth_dev *dev, 113 struct mlx5_mp_arg_queue_state_modify *sm); 114 115 static inline void 116 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *restrict tcp, 117 volatile struct mlx5_cqe *restrict cqe, 118 uint32_t phcsum); 119 120 static inline void 121 mlx5_lro_update_hdr(uint8_t *restrict padd, 122 volatile struct mlx5_cqe *restrict cqe, 123 uint32_t len); 124 125 uint32_t mlx5_ptype_table[] __rte_cache_aligned = { 126 [0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */ 127 }; 128 129 uint8_t mlx5_cksum_table[1 << 10] __rte_cache_aligned; 130 uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned; 131 132 uint64_t rte_net_mlx5_dynf_inline_mask; 133 #define PKT_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask 134 135 /** 136 * Build a table to translate Rx completion flags to packet type. 137 * 138 * @note: fix mlx5_dev_supported_ptypes_get() if any change here. 139 */ 140 void 141 mlx5_set_ptype_table(void) 142 { 143 unsigned int i; 144 uint32_t (*p)[RTE_DIM(mlx5_ptype_table)] = &mlx5_ptype_table; 145 146 /* Last entry must not be overwritten, reserved for errored packet. */ 147 for (i = 0; i < RTE_DIM(mlx5_ptype_table) - 1; ++i) 148 (*p)[i] = RTE_PTYPE_UNKNOWN; 149 /* 150 * The index to the array should have: 151 * bit[1:0] = l3_hdr_type 152 * bit[4:2] = l4_hdr_type 153 * bit[5] = ip_frag 154 * bit[6] = tunneled 155 * bit[7] = outer_l3_type 156 */ 157 /* L2 */ 158 (*p)[0x00] = RTE_PTYPE_L2_ETHER; 159 /* L3 */ 160 (*p)[0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 161 RTE_PTYPE_L4_NONFRAG; 162 (*p)[0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 163 RTE_PTYPE_L4_NONFRAG; 164 /* Fragmented */ 165 (*p)[0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 166 RTE_PTYPE_L4_FRAG; 167 (*p)[0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 168 RTE_PTYPE_L4_FRAG; 169 /* TCP */ 170 (*p)[0x05] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 171 RTE_PTYPE_L4_TCP; 172 (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 173 RTE_PTYPE_L4_TCP; 174 (*p)[0x0d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 175 RTE_PTYPE_L4_TCP; 176 (*p)[0x0e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 177 RTE_PTYPE_L4_TCP; 178 (*p)[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 179 RTE_PTYPE_L4_TCP; 180 (*p)[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 181 RTE_PTYPE_L4_TCP; 182 /* UDP */ 183 (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 184 RTE_PTYPE_L4_UDP; 185 (*p)[0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 186 RTE_PTYPE_L4_UDP; 187 /* Repeat with outer_l3_type being set. Just in case. */ 188 (*p)[0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 189 RTE_PTYPE_L4_NONFRAG; 190 (*p)[0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 191 RTE_PTYPE_L4_NONFRAG; 192 (*p)[0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 193 RTE_PTYPE_L4_FRAG; 194 (*p)[0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 195 RTE_PTYPE_L4_FRAG; 196 (*p)[0x85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 197 RTE_PTYPE_L4_TCP; 198 (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 199 RTE_PTYPE_L4_TCP; 200 (*p)[0x8d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 201 RTE_PTYPE_L4_TCP; 202 (*p)[0x8e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 203 RTE_PTYPE_L4_TCP; 204 (*p)[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 205 RTE_PTYPE_L4_TCP; 206 (*p)[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 207 RTE_PTYPE_L4_TCP; 208 (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 209 RTE_PTYPE_L4_UDP; 210 (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 211 RTE_PTYPE_L4_UDP; 212 /* Tunneled - L3 */ 213 (*p)[0x40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 214 (*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 215 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 216 RTE_PTYPE_INNER_L4_NONFRAG; 217 (*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 218 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 219 RTE_PTYPE_INNER_L4_NONFRAG; 220 (*p)[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 221 (*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 222 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 223 RTE_PTYPE_INNER_L4_NONFRAG; 224 (*p)[0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 225 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 226 RTE_PTYPE_INNER_L4_NONFRAG; 227 /* Tunneled - Fragmented */ 228 (*p)[0x61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 229 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 230 RTE_PTYPE_INNER_L4_FRAG; 231 (*p)[0x62] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 232 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 233 RTE_PTYPE_INNER_L4_FRAG; 234 (*p)[0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 235 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 236 RTE_PTYPE_INNER_L4_FRAG; 237 (*p)[0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 238 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 239 RTE_PTYPE_INNER_L4_FRAG; 240 /* Tunneled - TCP */ 241 (*p)[0x45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 242 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 243 RTE_PTYPE_INNER_L4_TCP; 244 (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 245 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 246 RTE_PTYPE_INNER_L4_TCP; 247 (*p)[0x4d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 248 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 249 RTE_PTYPE_INNER_L4_TCP; 250 (*p)[0x4e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 251 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 252 RTE_PTYPE_INNER_L4_TCP; 253 (*p)[0x51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 254 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 255 RTE_PTYPE_INNER_L4_TCP; 256 (*p)[0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 257 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 258 RTE_PTYPE_INNER_L4_TCP; 259 (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 260 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 261 RTE_PTYPE_INNER_L4_TCP; 262 (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 263 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 264 RTE_PTYPE_INNER_L4_TCP; 265 (*p)[0xcd] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 266 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 267 RTE_PTYPE_INNER_L4_TCP; 268 (*p)[0xce] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 269 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 270 RTE_PTYPE_INNER_L4_TCP; 271 (*p)[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 272 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 273 RTE_PTYPE_INNER_L4_TCP; 274 (*p)[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 275 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 276 RTE_PTYPE_INNER_L4_TCP; 277 /* Tunneled - UDP */ 278 (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 279 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 280 RTE_PTYPE_INNER_L4_UDP; 281 (*p)[0x4a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 282 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 283 RTE_PTYPE_INNER_L4_UDP; 284 (*p)[0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 285 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | 286 RTE_PTYPE_INNER_L4_UDP; 287 (*p)[0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | 288 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | 289 RTE_PTYPE_INNER_L4_UDP; 290 } 291 292 /** 293 * Build a table to translate packet to checksum type of Verbs. 294 */ 295 void 296 mlx5_set_cksum_table(void) 297 { 298 unsigned int i; 299 uint8_t v; 300 301 /* 302 * The index should have: 303 * bit[0] = PKT_TX_TCP_SEG 304 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM 305 * bit[4] = PKT_TX_IP_CKSUM 306 * bit[8] = PKT_TX_OUTER_IP_CKSUM 307 * bit[9] = tunnel 308 */ 309 for (i = 0; i < RTE_DIM(mlx5_cksum_table); ++i) { 310 v = 0; 311 if (i & (1 << 9)) { 312 /* Tunneled packet. */ 313 if (i & (1 << 8)) /* Outer IP. */ 314 v |= MLX5_ETH_WQE_L3_CSUM; 315 if (i & (1 << 4)) /* Inner IP. */ 316 v |= MLX5_ETH_WQE_L3_INNER_CSUM; 317 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */ 318 v |= MLX5_ETH_WQE_L4_INNER_CSUM; 319 } else { 320 /* No tunnel. */ 321 if (i & (1 << 4)) /* IP. */ 322 v |= MLX5_ETH_WQE_L3_CSUM; 323 if (i & (3 << 2 | 1 << 0)) /* L4 or TSO. */ 324 v |= MLX5_ETH_WQE_L4_CSUM; 325 } 326 mlx5_cksum_table[i] = v; 327 } 328 } 329 330 /** 331 * Build a table to translate packet type of mbuf to SWP type of Verbs. 332 */ 333 void 334 mlx5_set_swp_types_table(void) 335 { 336 unsigned int i; 337 uint8_t v; 338 339 /* 340 * The index should have: 341 * bit[0:1] = PKT_TX_L4_MASK 342 * bit[4] = PKT_TX_IPV6 343 * bit[8] = PKT_TX_OUTER_IPV6 344 * bit[9] = PKT_TX_OUTER_UDP 345 */ 346 for (i = 0; i < RTE_DIM(mlx5_swp_types_table); ++i) { 347 v = 0; 348 if (i & (1 << 8)) 349 v |= MLX5_ETH_WQE_L3_OUTER_IPV6; 350 if (i & (1 << 9)) 351 v |= MLX5_ETH_WQE_L4_OUTER_UDP; 352 if (i & (1 << 4)) 353 v |= MLX5_ETH_WQE_L3_INNER_IPV6; 354 if ((i & 3) == (PKT_TX_UDP_CKSUM >> 52)) 355 v |= MLX5_ETH_WQE_L4_INNER_UDP; 356 mlx5_swp_types_table[i] = v; 357 } 358 } 359 360 /** 361 * Set Software Parser flags and offsets in Ethernet Segment of WQE. 362 * Flags must be preliminary initialized to zero. 363 * 364 * @param loc 365 * Pointer to burst routine local context. 366 * @param swp_flags 367 * Pointer to store Software Parser flags 368 * @param olx 369 * Configured Tx offloads mask. It is fully defined at 370 * compile time and may be used for optimization. 371 * 372 * @return 373 * Software Parser offsets packed in dword. 374 * Software Parser flags are set by pointer. 375 */ 376 static __rte_always_inline uint32_t 377 txq_mbuf_to_swp(struct mlx5_txq_local *restrict loc, 378 uint8_t *swp_flags, 379 unsigned int olx) 380 { 381 uint64_t ol, tunnel; 382 unsigned int idx, off; 383 uint32_t set; 384 385 if (!MLX5_TXOFF_CONFIG(SWP)) 386 return 0; 387 ol = loc->mbuf->ol_flags; 388 tunnel = ol & PKT_TX_TUNNEL_MASK; 389 /* 390 * Check whether Software Parser is required. 391 * Only customized tunnels may ask for. 392 */ 393 if (likely(tunnel != PKT_TX_TUNNEL_UDP && tunnel != PKT_TX_TUNNEL_IP)) 394 return 0; 395 /* 396 * The index should have: 397 * bit[0:1] = PKT_TX_L4_MASK 398 * bit[4] = PKT_TX_IPV6 399 * bit[8] = PKT_TX_OUTER_IPV6 400 * bit[9] = PKT_TX_OUTER_UDP 401 */ 402 idx = (ol & (PKT_TX_L4_MASK | PKT_TX_IPV6 | PKT_TX_OUTER_IPV6)) >> 52; 403 idx |= (tunnel == PKT_TX_TUNNEL_UDP) ? (1 << 9) : 0; 404 *swp_flags = mlx5_swp_types_table[idx]; 405 /* 406 * Set offsets for SW parser. Since ConnectX-5, SW parser just 407 * complements HW parser. SW parser starts to engage only if HW parser 408 * can't reach a header. For the older devices, HW parser will not kick 409 * in if any of SWP offsets is set. Therefore, all of the L3 offsets 410 * should be set regardless of HW offload. 411 */ 412 off = loc->mbuf->outer_l2_len; 413 if (MLX5_TXOFF_CONFIG(VLAN) && ol & PKT_TX_VLAN_PKT) 414 off += sizeof(struct rte_vlan_hdr); 415 set = (off >> 1) << 8; /* Outer L3 offset. */ 416 off += loc->mbuf->outer_l3_len; 417 if (tunnel == PKT_TX_TUNNEL_UDP) 418 set |= off >> 1; /* Outer L4 offset. */ 419 if (ol & (PKT_TX_IPV4 | PKT_TX_IPV6)) { /* Inner IP. */ 420 const uint64_t csum = ol & PKT_TX_L4_MASK; 421 off += loc->mbuf->l2_len; 422 set |= (off >> 1) << 24; /* Inner L3 offset. */ 423 if (csum == PKT_TX_TCP_CKSUM || 424 csum == PKT_TX_UDP_CKSUM || 425 (MLX5_TXOFF_CONFIG(TSO) && ol & PKT_TX_TCP_SEG)) { 426 off += loc->mbuf->l3_len; 427 set |= (off >> 1) << 16; /* Inner L4 offset. */ 428 } 429 } 430 set = rte_cpu_to_le_32(set); 431 return set; 432 } 433 434 /** 435 * Convert the Checksum offloads to Verbs. 436 * 437 * @param buf 438 * Pointer to the mbuf. 439 * 440 * @return 441 * Converted checksum flags. 442 */ 443 static __rte_always_inline uint8_t 444 txq_ol_cksum_to_cs(struct rte_mbuf *buf) 445 { 446 uint32_t idx; 447 uint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK); 448 const uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK | 449 PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM; 450 451 /* 452 * The index should have: 453 * bit[0] = PKT_TX_TCP_SEG 454 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM 455 * bit[4] = PKT_TX_IP_CKSUM 456 * bit[8] = PKT_TX_OUTER_IP_CKSUM 457 * bit[9] = tunnel 458 */ 459 idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9); 460 return mlx5_cksum_table[idx]; 461 } 462 463 /** 464 * Internal function to compute the number of used descriptors in an RX queue 465 * 466 * @param rxq 467 * The Rx queue. 468 * 469 * @return 470 * The number of used rx descriptor. 471 */ 472 static uint32_t 473 rx_queue_count(struct mlx5_rxq_data *rxq) 474 { 475 struct rxq_zip *zip = &rxq->zip; 476 volatile struct mlx5_cqe *cqe; 477 const unsigned int cqe_n = (1 << rxq->cqe_n); 478 const unsigned int cqe_cnt = cqe_n - 1; 479 unsigned int cq_ci; 480 unsigned int used; 481 482 /* if we are processing a compressed cqe */ 483 if (zip->ai) { 484 used = zip->cqe_cnt - zip->ca; 485 cq_ci = zip->cq_ci; 486 } else { 487 used = 0; 488 cq_ci = rxq->cq_ci; 489 } 490 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; 491 while (check_cqe(cqe, cqe_n, cq_ci) != MLX5_CQE_STATUS_HW_OWN) { 492 int8_t op_own; 493 unsigned int n; 494 495 op_own = cqe->op_own; 496 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) 497 n = rte_be_to_cpu_32(cqe->byte_cnt); 498 else 499 n = 1; 500 cq_ci += n; 501 used += n; 502 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; 503 } 504 used = RTE_MIN(used, (1U << rxq->elts_n) - 1); 505 return used; 506 } 507 508 /** 509 * DPDK callback to check the status of a rx descriptor. 510 * 511 * @param rx_queue 512 * The Rx queue. 513 * @param[in] offset 514 * The index of the descriptor in the ring. 515 * 516 * @return 517 * The status of the tx descriptor. 518 */ 519 int 520 mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset) 521 { 522 struct mlx5_rxq_data *rxq = rx_queue; 523 struct mlx5_rxq_ctrl *rxq_ctrl = 524 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 525 struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv); 526 527 if (dev->rx_pkt_burst != mlx5_rx_burst) { 528 rte_errno = ENOTSUP; 529 return -rte_errno; 530 } 531 if (offset >= (1 << rxq->elts_n)) { 532 rte_errno = EINVAL; 533 return -rte_errno; 534 } 535 if (offset < rx_queue_count(rxq)) 536 return RTE_ETH_RX_DESC_DONE; 537 return RTE_ETH_RX_DESC_AVAIL; 538 } 539 540 /** 541 * DPDK callback to get the RX queue information 542 * 543 * @param dev 544 * Pointer to the device structure. 545 * 546 * @param rx_queue_id 547 * Rx queue identificator. 548 * 549 * @param qinfo 550 * Pointer to the RX queue information structure. 551 * 552 * @return 553 * None. 554 */ 555 556 void 557 mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, 558 struct rte_eth_rxq_info *qinfo) 559 { 560 struct mlx5_priv *priv = dev->data->dev_private; 561 struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id]; 562 struct mlx5_rxq_ctrl *rxq_ctrl = 563 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 564 565 if (!rxq) 566 return; 567 qinfo->mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 568 rxq->mprq_mp : rxq->mp; 569 qinfo->conf.rx_thresh.pthresh = 0; 570 qinfo->conf.rx_thresh.hthresh = 0; 571 qinfo->conf.rx_thresh.wthresh = 0; 572 qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh; 573 qinfo->conf.rx_drop_en = 1; 574 qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1; 575 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 576 qinfo->scattered_rx = dev->data->scattered_rx; 577 qinfo->nb_desc = 1 << rxq->elts_n; 578 } 579 580 /** 581 * DPDK callback to get the RX packet burst mode information 582 * 583 * @param dev 584 * Pointer to the device structure. 585 * 586 * @param rx_queue_id 587 * Rx queue identificatior. 588 * 589 * @param mode 590 * Pointer to the burts mode information. 591 * 592 * @return 593 * 0 as success, -EINVAL as failure. 594 */ 595 596 int 597 mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, 598 uint16_t rx_queue_id __rte_unused, 599 struct rte_eth_burst_mode *mode) 600 { 601 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 602 603 if (pkt_burst == mlx5_rx_burst) { 604 snprintf(mode->info, sizeof(mode->info), "%s", "Scalar"); 605 } else if (pkt_burst == mlx5_rx_burst_mprq) { 606 snprintf(mode->info, sizeof(mode->info), "%s", "Multi-Packet RQ"); 607 } else if (pkt_burst == mlx5_rx_burst_vec) { 608 #if defined RTE_ARCH_X86_64 609 snprintf(mode->info, sizeof(mode->info), "%s", "Vector SSE"); 610 #elif defined RTE_ARCH_ARM64 611 snprintf(mode->info, sizeof(mode->info), "%s", "Vector Neon"); 612 #elif defined RTE_ARCH_PPC_64 613 snprintf(mode->info, sizeof(mode->info), "%s", "Vector AltiVec"); 614 #else 615 return -EINVAL; 616 #endif 617 } else { 618 return -EINVAL; 619 } 620 return 0; 621 } 622 623 /** 624 * DPDK callback to get the number of used descriptors in a RX queue 625 * 626 * @param dev 627 * Pointer to the device structure. 628 * 629 * @param rx_queue_id 630 * The Rx queue. 631 * 632 * @return 633 * The number of used rx descriptor. 634 * -EINVAL if the queue is invalid 635 */ 636 uint32_t 637 mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) 638 { 639 struct mlx5_priv *priv = dev->data->dev_private; 640 struct mlx5_rxq_data *rxq; 641 642 if (dev->rx_pkt_burst != mlx5_rx_burst) { 643 rte_errno = ENOTSUP; 644 return -rte_errno; 645 } 646 rxq = (*priv->rxqs)[rx_queue_id]; 647 if (!rxq) { 648 rte_errno = EINVAL; 649 return -rte_errno; 650 } 651 return rx_queue_count(rxq); 652 } 653 654 #define MLX5_SYSTEM_LOG_DIR "/var/log" 655 /** 656 * Dump debug information to log file. 657 * 658 * @param fname 659 * The file name. 660 * @param hex_title 661 * If not NULL this string is printed as a header to the output 662 * and the output will be in hexadecimal view. 663 * @param buf 664 * This is the buffer address to print out. 665 * @param len 666 * The number of bytes to dump out. 667 */ 668 void 669 mlx5_dump_debug_information(const char *fname, const char *hex_title, 670 const void *buf, unsigned int hex_len) 671 { 672 FILE *fd; 673 674 MKSTR(path, "%s/%s", MLX5_SYSTEM_LOG_DIR, fname); 675 fd = fopen(path, "a+"); 676 if (!fd) { 677 DRV_LOG(WARNING, "cannot open %s for debug dump", path); 678 MKSTR(path2, "./%s", fname); 679 fd = fopen(path2, "a+"); 680 if (!fd) { 681 DRV_LOG(ERR, "cannot open %s for debug dump", path2); 682 return; 683 } 684 DRV_LOG(INFO, "New debug dump in file %s", path2); 685 } else { 686 DRV_LOG(INFO, "New debug dump in file %s", path); 687 } 688 if (hex_title) 689 rte_hexdump(fd, hex_title, buf, hex_len); 690 else 691 fprintf(fd, "%s", (const char *)buf); 692 fprintf(fd, "\n\n\n"); 693 fclose(fd); 694 } 695 696 /** 697 * Move QP from error state to running state and initialize indexes. 698 * 699 * @param txq_ctrl 700 * Pointer to TX queue control structure. 701 * 702 * @return 703 * 0 on success, else -1. 704 */ 705 static int 706 tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl) 707 { 708 struct mlx5_mp_arg_queue_state_modify sm = { 709 .is_wq = 0, 710 .queue_id = txq_ctrl->txq.idx, 711 }; 712 713 if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm)) 714 return -1; 715 txq_ctrl->txq.wqe_ci = 0; 716 txq_ctrl->txq.wqe_pi = 0; 717 txq_ctrl->txq.elts_comp = 0; 718 return 0; 719 } 720 721 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */ 722 static int 723 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe) 724 { 725 static const uint8_t magic[] = "seen"; 726 int ret = 1; 727 unsigned int i; 728 729 for (i = 0; i < sizeof(magic); ++i) 730 if (!ret || err_cqe->rsvd1[i] != magic[i]) { 731 ret = 0; 732 err_cqe->rsvd1[i] = magic[i]; 733 } 734 return ret; 735 } 736 737 /** 738 * Handle error CQE. 739 * 740 * @param txq 741 * Pointer to TX queue structure. 742 * @param error_cqe 743 * Pointer to the error CQE. 744 * 745 * @return 746 * Negative value if queue recovery failed, otherwise 747 * the error completion entry is handled successfully. 748 */ 749 static int 750 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *restrict txq, 751 volatile struct mlx5_err_cqe *err_cqe) 752 { 753 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) { 754 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1); 755 struct mlx5_txq_ctrl *txq_ctrl = 756 container_of(txq, struct mlx5_txq_ctrl, txq); 757 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter); 758 int seen = check_err_cqe_seen(err_cqe); 759 760 if (!seen && txq_ctrl->dump_file_n < 761 txq_ctrl->priv->config.max_dump_files_num) { 762 MKSTR(err_str, "Unexpected CQE error syndrome " 763 "0x%02x CQN = %u SQN = %u wqe_counter = %u " 764 "wq_ci = %u cq_ci = %u", err_cqe->syndrome, 765 txq->cqe_s, txq->qp_num_8s >> 8, 766 rte_be_to_cpu_16(err_cqe->wqe_counter), 767 txq->wqe_ci, txq->cq_ci); 768 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u", 769 PORT_ID(txq_ctrl->priv), txq->idx, 770 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc()); 771 mlx5_dump_debug_information(name, NULL, err_str, 0); 772 mlx5_dump_debug_information(name, "MLX5 Error CQ:", 773 (const void *)((uintptr_t) 774 txq->cqes), 775 sizeof(*err_cqe) * 776 (1 << txq->cqe_n)); 777 mlx5_dump_debug_information(name, "MLX5 Error SQ:", 778 (const void *)((uintptr_t) 779 txq->wqes), 780 MLX5_WQE_SIZE * 781 (1 << txq->wqe_n)); 782 txq_ctrl->dump_file_n++; 783 } 784 if (!seen) 785 /* 786 * Count errors in WQEs units. 787 * Later it can be improved to count error packets, 788 * for example, by SQ parsing to find how much packets 789 * should be counted for each WQE. 790 */ 791 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) - 792 new_wqe_pi) & wqe_m; 793 if (tx_recover_qp(txq_ctrl)) { 794 /* Recovering failed - retry later on the same WQE. */ 795 return -1; 796 } 797 /* Release all the remaining buffers. */ 798 txq_free_elts(txq_ctrl); 799 } 800 return 0; 801 } 802 803 /** 804 * Translate RX completion flags to packet type. 805 * 806 * @param[in] rxq 807 * Pointer to RX queue structure. 808 * @param[in] cqe 809 * Pointer to CQE. 810 * 811 * @note: fix mlx5_dev_supported_ptypes_get() if any change here. 812 * 813 * @return 814 * Packet type for struct rte_mbuf. 815 */ 816 static inline uint32_t 817 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe) 818 { 819 uint8_t idx; 820 uint8_t pinfo = cqe->pkt_info; 821 uint16_t ptype = cqe->hdr_type_etc; 822 823 /* 824 * The index to the array should have: 825 * bit[1:0] = l3_hdr_type 826 * bit[4:2] = l4_hdr_type 827 * bit[5] = ip_frag 828 * bit[6] = tunneled 829 * bit[7] = outer_l3_type 830 */ 831 idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10); 832 return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6)); 833 } 834 835 /** 836 * Initialize Rx WQ and indexes. 837 * 838 * @param[in] rxq 839 * Pointer to RX queue structure. 840 */ 841 void 842 mlx5_rxq_initialize(struct mlx5_rxq_data *rxq) 843 { 844 const unsigned int wqe_n = 1 << rxq->elts_n; 845 unsigned int i; 846 847 for (i = 0; (i != wqe_n); ++i) { 848 volatile struct mlx5_wqe_data_seg *scat; 849 uintptr_t addr; 850 uint32_t byte_count; 851 852 if (mlx5_rxq_mprq_enabled(rxq)) { 853 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i]; 854 855 scat = &((volatile struct mlx5_wqe_mprq *) 856 rxq->wqes)[i].dseg; 857 addr = (uintptr_t)mlx5_mprq_buf_addr(buf, 858 1 << rxq->strd_num_n); 859 byte_count = (1 << rxq->strd_sz_n) * 860 (1 << rxq->strd_num_n); 861 } else { 862 struct rte_mbuf *buf = (*rxq->elts)[i]; 863 864 scat = &((volatile struct mlx5_wqe_data_seg *) 865 rxq->wqes)[i]; 866 addr = rte_pktmbuf_mtod(buf, uintptr_t); 867 byte_count = DATA_LEN(buf); 868 } 869 /* scat->addr must be able to store a pointer. */ 870 MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t)); 871 *scat = (struct mlx5_wqe_data_seg){ 872 .addr = rte_cpu_to_be_64(addr), 873 .byte_count = rte_cpu_to_be_32(byte_count), 874 .lkey = mlx5_rx_addr2mr(rxq, addr), 875 }; 876 } 877 rxq->consumed_strd = 0; 878 rxq->decompressed = 0; 879 rxq->rq_pi = 0; 880 rxq->zip = (struct rxq_zip){ 881 .ai = 0, 882 }; 883 /* Update doorbell counter. */ 884 rxq->rq_ci = wqe_n >> rxq->sges_n; 885 rte_cio_wmb(); 886 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 887 } 888 889 /** 890 * Modify a Verbs/DevX queue state. 891 * This must be called from the primary process. 892 * 893 * @param dev 894 * Pointer to Ethernet device. 895 * @param sm 896 * State modify request parameters. 897 * 898 * @return 899 * 0 in case of success else non-zero value and rte_errno is set. 900 */ 901 int 902 mlx5_queue_state_modify_primary(struct rte_eth_dev *dev, 903 const struct mlx5_mp_arg_queue_state_modify *sm) 904 { 905 int ret; 906 struct mlx5_priv *priv = dev->data->dev_private; 907 908 if (sm->is_wq) { 909 struct mlx5_rxq_data *rxq = (*priv->rxqs)[sm->queue_id]; 910 struct mlx5_rxq_ctrl *rxq_ctrl = 911 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 912 913 if (rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_IBV) { 914 struct ibv_wq_attr mod = { 915 .attr_mask = IBV_WQ_ATTR_STATE, 916 .wq_state = sm->state, 917 }; 918 919 ret = mlx5_glue->modify_wq(rxq_ctrl->obj->wq, &mod); 920 } else { /* rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ. */ 921 struct mlx5_devx_modify_rq_attr rq_attr; 922 923 memset(&rq_attr, 0, sizeof(rq_attr)); 924 if (sm->state == IBV_WQS_RESET) { 925 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 926 rq_attr.state = MLX5_RQC_STATE_RST; 927 } else if (sm->state == IBV_WQS_RDY) { 928 rq_attr.rq_state = MLX5_RQC_STATE_RST; 929 rq_attr.state = MLX5_RQC_STATE_RDY; 930 } else if (sm->state == IBV_WQS_ERR) { 931 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 932 rq_attr.state = MLX5_RQC_STATE_ERR; 933 } 934 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, 935 &rq_attr); 936 } 937 if (ret) { 938 DRV_LOG(ERR, "Cannot change Rx WQ state to %u - %s", 939 sm->state, strerror(errno)); 940 rte_errno = errno; 941 return ret; 942 } 943 } else { 944 struct mlx5_txq_data *txq = (*priv->txqs)[sm->queue_id]; 945 struct mlx5_txq_ctrl *txq_ctrl = 946 container_of(txq, struct mlx5_txq_ctrl, txq); 947 struct ibv_qp_attr mod = { 948 .qp_state = IBV_QPS_RESET, 949 .port_num = (uint8_t)priv->ibv_port, 950 }; 951 struct ibv_qp *qp = txq_ctrl->obj->qp; 952 953 ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE); 954 if (ret) { 955 DRV_LOG(ERR, "Cannot change the Tx QP state to RESET " 956 "%s", strerror(errno)); 957 rte_errno = errno; 958 return ret; 959 } 960 mod.qp_state = IBV_QPS_INIT; 961 ret = mlx5_glue->modify_qp(qp, &mod, 962 (IBV_QP_STATE | IBV_QP_PORT)); 963 if (ret) { 964 DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s", 965 strerror(errno)); 966 rte_errno = errno; 967 return ret; 968 } 969 mod.qp_state = IBV_QPS_RTR; 970 ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE); 971 if (ret) { 972 DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s", 973 strerror(errno)); 974 rte_errno = errno; 975 return ret; 976 } 977 mod.qp_state = IBV_QPS_RTS; 978 ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE); 979 if (ret) { 980 DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s", 981 strerror(errno)); 982 rte_errno = errno; 983 return ret; 984 } 985 } 986 return 0; 987 } 988 989 /** 990 * Modify a Verbs queue state. 991 * 992 * @param dev 993 * Pointer to Ethernet device. 994 * @param sm 995 * State modify request parameters. 996 * 997 * @return 998 * 0 in case of success else non-zero value. 999 */ 1000 static int 1001 mlx5_queue_state_modify(struct rte_eth_dev *dev, 1002 struct mlx5_mp_arg_queue_state_modify *sm) 1003 { 1004 struct mlx5_priv *priv = dev->data->dev_private; 1005 int ret = 0; 1006 1007 switch (rte_eal_process_type()) { 1008 case RTE_PROC_PRIMARY: 1009 ret = mlx5_queue_state_modify_primary(dev, sm); 1010 break; 1011 case RTE_PROC_SECONDARY: 1012 ret = mlx5_mp_req_queue_state_modify(&priv->mp_id, sm); 1013 break; 1014 default: 1015 break; 1016 } 1017 return ret; 1018 } 1019 1020 /** 1021 * Handle a Rx error. 1022 * The function inserts the RQ state to reset when the first error CQE is 1023 * shown, then drains the CQ by the caller function loop. When the CQ is empty, 1024 * it moves the RQ state to ready and initializes the RQ. 1025 * Next CQE identification and error counting are in the caller responsibility. 1026 * 1027 * @param[in] rxq 1028 * Pointer to RX queue structure. 1029 * @param[in] vec 1030 * 1 when called from vectorized Rx burst, need to prepare mbufs for the RQ. 1031 * 0 when called from non-vectorized Rx burst. 1032 * 1033 * @return 1034 * -1 in case of recovery error, otherwise the CQE status. 1035 */ 1036 int 1037 mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) 1038 { 1039 const uint16_t cqe_n = 1 << rxq->cqe_n; 1040 const uint16_t cqe_mask = cqe_n - 1; 1041 const unsigned int wqe_n = 1 << rxq->elts_n; 1042 struct mlx5_rxq_ctrl *rxq_ctrl = 1043 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 1044 union { 1045 volatile struct mlx5_cqe *cqe; 1046 volatile struct mlx5_err_cqe *err_cqe; 1047 } u = { 1048 .cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask], 1049 }; 1050 struct mlx5_mp_arg_queue_state_modify sm; 1051 int ret; 1052 1053 switch (rxq->err_state) { 1054 case MLX5_RXQ_ERR_STATE_NO_ERROR: 1055 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET; 1056 /* Fall-through */ 1057 case MLX5_RXQ_ERR_STATE_NEED_RESET: 1058 sm.is_wq = 1; 1059 sm.queue_id = rxq->idx; 1060 sm.state = IBV_WQS_RESET; 1061 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), &sm)) 1062 return -1; 1063 if (rxq_ctrl->dump_file_n < 1064 rxq_ctrl->priv->config.max_dump_files_num) { 1065 MKSTR(err_str, "Unexpected CQE error syndrome " 1066 "0x%02x CQN = %u RQN = %u wqe_counter = %u" 1067 " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome, 1068 rxq->cqn, rxq_ctrl->wqn, 1069 rte_be_to_cpu_16(u.err_cqe->wqe_counter), 1070 rxq->rq_ci << rxq->sges_n, rxq->cq_ci); 1071 MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u", 1072 rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc()); 1073 mlx5_dump_debug_information(name, NULL, err_str, 0); 1074 mlx5_dump_debug_information(name, "MLX5 Error CQ:", 1075 (const void *)((uintptr_t) 1076 rxq->cqes), 1077 sizeof(*u.cqe) * cqe_n); 1078 mlx5_dump_debug_information(name, "MLX5 Error RQ:", 1079 (const void *)((uintptr_t) 1080 rxq->wqes), 1081 16 * wqe_n); 1082 rxq_ctrl->dump_file_n++; 1083 } 1084 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY; 1085 /* Fall-through */ 1086 case MLX5_RXQ_ERR_STATE_NEED_READY: 1087 ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci); 1088 if (ret == MLX5_CQE_STATUS_HW_OWN) { 1089 rte_cio_wmb(); 1090 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1091 rte_cio_wmb(); 1092 /* 1093 * The RQ consumer index must be zeroed while moving 1094 * from RESET state to RDY state. 1095 */ 1096 *rxq->rq_db = rte_cpu_to_be_32(0); 1097 rte_cio_wmb(); 1098 sm.is_wq = 1; 1099 sm.queue_id = rxq->idx; 1100 sm.state = IBV_WQS_RDY; 1101 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), 1102 &sm)) 1103 return -1; 1104 if (vec) { 1105 const uint16_t q_mask = wqe_n - 1; 1106 uint16_t elt_idx; 1107 struct rte_mbuf **elt; 1108 int i; 1109 unsigned int n = wqe_n - (rxq->rq_ci - 1110 rxq->rq_pi); 1111 1112 for (i = 0; i < (int)n; ++i) { 1113 elt_idx = (rxq->rq_ci + i) & q_mask; 1114 elt = &(*rxq->elts)[elt_idx]; 1115 *elt = rte_mbuf_raw_alloc(rxq->mp); 1116 if (!*elt) { 1117 for (i--; i >= 0; --i) { 1118 elt_idx = (rxq->rq_ci + 1119 i) & q_mask; 1120 elt = &(*rxq->elts) 1121 [elt_idx]; 1122 rte_pktmbuf_free_seg 1123 (*elt); 1124 } 1125 return -1; 1126 } 1127 } 1128 for (i = 0; i < (int)wqe_n; ++i) { 1129 elt = &(*rxq->elts)[i]; 1130 DATA_LEN(*elt) = 1131 (uint16_t)((*elt)->buf_len - 1132 rte_pktmbuf_headroom(*elt)); 1133 } 1134 /* Padding with a fake mbuf for vec Rx. */ 1135 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 1136 (*rxq->elts)[wqe_n + i] = 1137 &rxq->fake_mbuf; 1138 } 1139 mlx5_rxq_initialize(rxq); 1140 rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; 1141 } 1142 return ret; 1143 default: 1144 return -1; 1145 } 1146 } 1147 1148 /** 1149 * Get size of the next packet for a given CQE. For compressed CQEs, the 1150 * consumer index is updated only once all packets of the current one have 1151 * been processed. 1152 * 1153 * @param rxq 1154 * Pointer to RX queue. 1155 * @param cqe 1156 * CQE to process. 1157 * @param[out] mcqe 1158 * Store pointer to mini-CQE if compressed. Otherwise, the pointer is not 1159 * written. 1160 * 1161 * @return 1162 * 0 in case of empty CQE, otherwise the packet size in bytes. 1163 */ 1164 static inline int 1165 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 1166 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe) 1167 { 1168 struct rxq_zip *zip = &rxq->zip; 1169 uint16_t cqe_n = cqe_cnt + 1; 1170 int len; 1171 uint16_t idx, end; 1172 1173 do { 1174 len = 0; 1175 /* Process compressed data in the CQE and mini arrays. */ 1176 if (zip->ai) { 1177 volatile struct mlx5_mini_cqe8 (*mc)[8] = 1178 (volatile struct mlx5_mini_cqe8 (*)[8]) 1179 (uintptr_t)(&(*rxq->cqes)[zip->ca & 1180 cqe_cnt].pkt_info); 1181 1182 len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt); 1183 *mcqe = &(*mc)[zip->ai & 7]; 1184 if ((++zip->ai & 7) == 0) { 1185 /* Invalidate consumed CQEs */ 1186 idx = zip->ca; 1187 end = zip->na; 1188 while (idx != end) { 1189 (*rxq->cqes)[idx & cqe_cnt].op_own = 1190 MLX5_CQE_INVALIDATE; 1191 ++idx; 1192 } 1193 /* 1194 * Increment consumer index to skip the number 1195 * of CQEs consumed. Hardware leaves holes in 1196 * the CQ ring for software use. 1197 */ 1198 zip->ca = zip->na; 1199 zip->na += 8; 1200 } 1201 if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) { 1202 /* Invalidate the rest */ 1203 idx = zip->ca; 1204 end = zip->cq_ci; 1205 1206 while (idx != end) { 1207 (*rxq->cqes)[idx & cqe_cnt].op_own = 1208 MLX5_CQE_INVALIDATE; 1209 ++idx; 1210 } 1211 rxq->cq_ci = zip->cq_ci; 1212 zip->ai = 0; 1213 } 1214 /* 1215 * No compressed data, get next CQE and verify if it is 1216 * compressed. 1217 */ 1218 } else { 1219 int ret; 1220 int8_t op_own; 1221 1222 ret = check_cqe(cqe, cqe_n, rxq->cq_ci); 1223 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 1224 if (unlikely(ret == MLX5_CQE_STATUS_ERR || 1225 rxq->err_state)) { 1226 ret = mlx5_rx_err_handle(rxq, 0); 1227 if (ret == MLX5_CQE_STATUS_HW_OWN || 1228 ret == -1) 1229 return 0; 1230 } else { 1231 return 0; 1232 } 1233 } 1234 ++rxq->cq_ci; 1235 op_own = cqe->op_own; 1236 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) { 1237 volatile struct mlx5_mini_cqe8 (*mc)[8] = 1238 (volatile struct mlx5_mini_cqe8 (*)[8]) 1239 (uintptr_t)(&(*rxq->cqes) 1240 [rxq->cq_ci & 1241 cqe_cnt].pkt_info); 1242 1243 /* Fix endianness. */ 1244 zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt); 1245 /* 1246 * Current mini array position is the one 1247 * returned by check_cqe64(). 1248 * 1249 * If completion comprises several mini arrays, 1250 * as a special case the second one is located 1251 * 7 CQEs after the initial CQE instead of 8 1252 * for subsequent ones. 1253 */ 1254 zip->ca = rxq->cq_ci; 1255 zip->na = zip->ca + 7; 1256 /* Compute the next non compressed CQE. */ 1257 --rxq->cq_ci; 1258 zip->cq_ci = rxq->cq_ci + zip->cqe_cnt; 1259 /* Get packet size to return. */ 1260 len = rte_be_to_cpu_32((*mc)[0].byte_cnt); 1261 *mcqe = &(*mc)[0]; 1262 zip->ai = 1; 1263 /* Prefetch all to be invalidated */ 1264 idx = zip->ca; 1265 end = zip->cq_ci; 1266 while (idx != end) { 1267 rte_prefetch0(&(*rxq->cqes)[(idx) & 1268 cqe_cnt]); 1269 ++idx; 1270 } 1271 } else { 1272 len = rte_be_to_cpu_32(cqe->byte_cnt); 1273 } 1274 } 1275 if (unlikely(rxq->err_state)) { 1276 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 1277 ++rxq->stats.idropped; 1278 } else { 1279 return len; 1280 } 1281 } while (1); 1282 } 1283 1284 /** 1285 * Translate RX completion flags to offload flags. 1286 * 1287 * @param[in] cqe 1288 * Pointer to CQE. 1289 * 1290 * @return 1291 * Offload flags (ol_flags) for struct rte_mbuf. 1292 */ 1293 static inline uint32_t 1294 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe) 1295 { 1296 uint32_t ol_flags = 0; 1297 uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc); 1298 1299 ol_flags = 1300 TRANSPOSE(flags, 1301 MLX5_CQE_RX_L3_HDR_VALID, 1302 PKT_RX_IP_CKSUM_GOOD) | 1303 TRANSPOSE(flags, 1304 MLX5_CQE_RX_L4_HDR_VALID, 1305 PKT_RX_L4_CKSUM_GOOD); 1306 return ol_flags; 1307 } 1308 1309 /** 1310 * Fill in mbuf fields from RX completion flags. 1311 * Note that pkt->ol_flags should be initialized outside of this function. 1312 * 1313 * @param rxq 1314 * Pointer to RX queue. 1315 * @param pkt 1316 * mbuf to fill. 1317 * @param cqe 1318 * CQE to process. 1319 * @param rss_hash_res 1320 * Packet RSS Hash result. 1321 */ 1322 static inline void 1323 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, 1324 volatile struct mlx5_cqe *cqe, uint32_t rss_hash_res) 1325 { 1326 /* Update packet information. */ 1327 pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe); 1328 if (rss_hash_res && rxq->rss_hash) { 1329 pkt->hash.rss = rss_hash_res; 1330 pkt->ol_flags |= PKT_RX_RSS_HASH; 1331 } 1332 if (rxq->mark && MLX5_FLOW_MARK_IS_VALID(cqe->sop_drop_qpn)) { 1333 pkt->ol_flags |= PKT_RX_FDIR; 1334 if (cqe->sop_drop_qpn != 1335 rte_cpu_to_be_32(MLX5_FLOW_MARK_DEFAULT)) { 1336 uint32_t mark = cqe->sop_drop_qpn; 1337 1338 pkt->ol_flags |= PKT_RX_FDIR_ID; 1339 pkt->hash.fdir.hi = mlx5_flow_mark_get(mark); 1340 } 1341 } 1342 if (rxq->dynf_meta && cqe->flow_table_metadata) { 1343 pkt->ol_flags |= rxq->flow_meta_mask; 1344 *RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset, uint32_t *) = 1345 cqe->flow_table_metadata; 1346 } 1347 if (rxq->csum) 1348 pkt->ol_flags |= rxq_cq_to_ol_flags(cqe); 1349 if (rxq->vlan_strip && 1350 (cqe->hdr_type_etc & rte_cpu_to_be_16(MLX5_CQE_VLAN_STRIPPED))) { 1351 pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 1352 pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info); 1353 } 1354 if (rxq->hw_timestamp) { 1355 pkt->timestamp = rte_be_to_cpu_64(cqe->timestamp); 1356 pkt->ol_flags |= PKT_RX_TIMESTAMP; 1357 } 1358 } 1359 1360 /** 1361 * DPDK callback for RX. 1362 * 1363 * @param dpdk_rxq 1364 * Generic pointer to RX queue structure. 1365 * @param[out] pkts 1366 * Array to store received packets. 1367 * @param pkts_n 1368 * Maximum number of packets in array. 1369 * 1370 * @return 1371 * Number of packets successfully received (<= pkts_n). 1372 */ 1373 uint16_t 1374 mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 1375 { 1376 struct mlx5_rxq_data *rxq = dpdk_rxq; 1377 const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1; 1378 const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1; 1379 const unsigned int sges_n = rxq->sges_n; 1380 struct rte_mbuf *pkt = NULL; 1381 struct rte_mbuf *seg = NULL; 1382 volatile struct mlx5_cqe *cqe = 1383 &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 1384 unsigned int i = 0; 1385 unsigned int rq_ci = rxq->rq_ci << sges_n; 1386 int len = 0; /* keep its value across iterations. */ 1387 1388 while (pkts_n) { 1389 unsigned int idx = rq_ci & wqe_cnt; 1390 volatile struct mlx5_wqe_data_seg *wqe = 1391 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx]; 1392 struct rte_mbuf *rep = (*rxq->elts)[idx]; 1393 volatile struct mlx5_mini_cqe8 *mcqe = NULL; 1394 uint32_t rss_hash_res; 1395 1396 if (pkt) 1397 NEXT(seg) = rep; 1398 seg = rep; 1399 rte_prefetch0(seg); 1400 rte_prefetch0(cqe); 1401 rte_prefetch0(wqe); 1402 rep = rte_mbuf_raw_alloc(rxq->mp); 1403 if (unlikely(rep == NULL)) { 1404 ++rxq->stats.rx_nombuf; 1405 if (!pkt) { 1406 /* 1407 * no buffers before we even started, 1408 * bail out silently. 1409 */ 1410 break; 1411 } 1412 while (pkt != seg) { 1413 MLX5_ASSERT(pkt != (*rxq->elts)[idx]); 1414 rep = NEXT(pkt); 1415 NEXT(pkt) = NULL; 1416 NB_SEGS(pkt) = 1; 1417 rte_mbuf_raw_free(pkt); 1418 pkt = rep; 1419 } 1420 break; 1421 } 1422 if (!pkt) { 1423 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 1424 len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe); 1425 if (!len) { 1426 rte_mbuf_raw_free(rep); 1427 break; 1428 } 1429 pkt = seg; 1430 MLX5_ASSERT(len >= (rxq->crc_present << 2)); 1431 pkt->ol_flags &= EXT_ATTACHED_MBUF; 1432 /* If compressed, take hash result from mini-CQE. */ 1433 rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ? 1434 cqe->rx_hash_res : 1435 mcqe->rx_hash_result); 1436 rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res); 1437 if (rxq->crc_present) 1438 len -= RTE_ETHER_CRC_LEN; 1439 PKT_LEN(pkt) = len; 1440 if (cqe->lro_num_seg > 1) { 1441 mlx5_lro_update_hdr 1442 (rte_pktmbuf_mtod(pkt, uint8_t *), cqe, 1443 len); 1444 pkt->ol_flags |= PKT_RX_LRO; 1445 pkt->tso_segsz = len / cqe->lro_num_seg; 1446 } 1447 } 1448 DATA_LEN(rep) = DATA_LEN(seg); 1449 PKT_LEN(rep) = PKT_LEN(seg); 1450 SET_DATA_OFF(rep, DATA_OFF(seg)); 1451 PORT(rep) = PORT(seg); 1452 (*rxq->elts)[idx] = rep; 1453 /* 1454 * Fill NIC descriptor with the new buffer. The lkey and size 1455 * of the buffers are already known, only the buffer address 1456 * changes. 1457 */ 1458 wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t)); 1459 /* If there's only one MR, no need to replace LKey in WQE. */ 1460 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) 1461 wqe->lkey = mlx5_rx_mb2mr(rxq, rep); 1462 if (len > DATA_LEN(seg)) { 1463 len -= DATA_LEN(seg); 1464 ++NB_SEGS(pkt); 1465 ++rq_ci; 1466 continue; 1467 } 1468 DATA_LEN(seg) = len; 1469 #ifdef MLX5_PMD_SOFT_COUNTERS 1470 /* Increment bytes counter. */ 1471 rxq->stats.ibytes += PKT_LEN(pkt); 1472 #endif 1473 /* Return packet. */ 1474 *(pkts++) = pkt; 1475 pkt = NULL; 1476 --pkts_n; 1477 ++i; 1478 /* Align consumer index to the next stride. */ 1479 rq_ci >>= sges_n; 1480 ++rq_ci; 1481 rq_ci <<= sges_n; 1482 } 1483 if (unlikely((i == 0) && ((rq_ci >> sges_n) == rxq->rq_ci))) 1484 return 0; 1485 /* Update the consumer index. */ 1486 rxq->rq_ci = rq_ci >> sges_n; 1487 rte_cio_wmb(); 1488 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1489 rte_cio_wmb(); 1490 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 1491 #ifdef MLX5_PMD_SOFT_COUNTERS 1492 /* Increment packets counter. */ 1493 rxq->stats.ipackets += i; 1494 #endif 1495 return i; 1496 } 1497 1498 /** 1499 * Update LRO packet TCP header. 1500 * The HW LRO feature doesn't update the TCP header after coalescing the 1501 * TCP segments but supplies information in CQE to fill it by SW. 1502 * 1503 * @param tcp 1504 * Pointer to the TCP header. 1505 * @param cqe 1506 * Pointer to the completion entry.. 1507 * @param phcsum 1508 * The L3 pseudo-header checksum. 1509 */ 1510 static inline void 1511 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *restrict tcp, 1512 volatile struct mlx5_cqe *restrict cqe, 1513 uint32_t phcsum) 1514 { 1515 uint8_t l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) & 1516 MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT; 1517 /* 1518 * The HW calculates only the TCP payload checksum, need to complete 1519 * the TCP header checksum and the L3 pseudo-header checksum. 1520 */ 1521 uint32_t csum = phcsum + cqe->csum; 1522 1523 if (l4_type == MLX5_L4_HDR_TYPE_TCP_EMPTY_ACK || 1524 l4_type == MLX5_L4_HDR_TYPE_TCP_WITH_ACL) { 1525 tcp->tcp_flags |= RTE_TCP_ACK_FLAG; 1526 tcp->recv_ack = cqe->lro_ack_seq_num; 1527 tcp->rx_win = cqe->lro_tcp_win; 1528 } 1529 if (cqe->lro_tcppsh_abort_dupack & MLX5_CQE_LRO_PUSH_MASK) 1530 tcp->tcp_flags |= RTE_TCP_PSH_FLAG; 1531 tcp->cksum = 0; 1532 csum += rte_raw_cksum(tcp, (tcp->data_off & 0xF) * 4); 1533 csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff); 1534 csum = (~csum) & 0xffff; 1535 if (csum == 0) 1536 csum = 0xffff; 1537 tcp->cksum = csum; 1538 } 1539 1540 /** 1541 * Update LRO packet headers. 1542 * The HW LRO feature doesn't update the L3/TCP headers after coalescing the 1543 * TCP segments but supply information in CQE to fill it by SW. 1544 * 1545 * @param padd 1546 * The packet address. 1547 * @param cqe 1548 * Pointer to the completion entry.. 1549 * @param len 1550 * The packet length. 1551 */ 1552 static inline void 1553 mlx5_lro_update_hdr(uint8_t *restrict padd, 1554 volatile struct mlx5_cqe *restrict cqe, 1555 uint32_t len) 1556 { 1557 union { 1558 struct rte_ether_hdr *eth; 1559 struct rte_vlan_hdr *vlan; 1560 struct rte_ipv4_hdr *ipv4; 1561 struct rte_ipv6_hdr *ipv6; 1562 struct rte_tcp_hdr *tcp; 1563 uint8_t *hdr; 1564 } h = { 1565 .hdr = padd, 1566 }; 1567 uint16_t proto = h.eth->ether_type; 1568 uint32_t phcsum; 1569 1570 h.eth++; 1571 while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) || 1572 proto == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { 1573 proto = h.vlan->eth_proto; 1574 h.vlan++; 1575 } 1576 if (proto == RTE_BE16(RTE_ETHER_TYPE_IPV4)) { 1577 h.ipv4->time_to_live = cqe->lro_min_ttl; 1578 h.ipv4->total_length = rte_cpu_to_be_16(len - (h.hdr - padd)); 1579 h.ipv4->hdr_checksum = 0; 1580 h.ipv4->hdr_checksum = rte_ipv4_cksum(h.ipv4); 1581 phcsum = rte_ipv4_phdr_cksum(h.ipv4, 0); 1582 h.ipv4++; 1583 } else { 1584 h.ipv6->hop_limits = cqe->lro_min_ttl; 1585 h.ipv6->payload_len = rte_cpu_to_be_16(len - (h.hdr - padd) - 1586 sizeof(*h.ipv6)); 1587 phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0); 1588 h.ipv6++; 1589 } 1590 mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum); 1591 } 1592 1593 void 1594 mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque) 1595 { 1596 struct mlx5_mprq_buf *buf = opaque; 1597 1598 if (rte_atomic16_read(&buf->refcnt) == 1) { 1599 rte_mempool_put(buf->mp, buf); 1600 } else if (rte_atomic16_add_return(&buf->refcnt, -1) == 0) { 1601 rte_atomic16_set(&buf->refcnt, 1); 1602 rte_mempool_put(buf->mp, buf); 1603 } 1604 } 1605 1606 void 1607 mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf) 1608 { 1609 mlx5_mprq_buf_free_cb(NULL, buf); 1610 } 1611 1612 static inline void 1613 mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx, 1614 const unsigned int strd_n) 1615 { 1616 struct mlx5_mprq_buf *rep = rxq->mprq_repl; 1617 volatile struct mlx5_wqe_data_seg *wqe = 1618 &((volatile struct mlx5_wqe_mprq *)rxq->wqes)[rq_idx].dseg; 1619 void *addr; 1620 1621 MLX5_ASSERT(rep != NULL); 1622 /* Replace MPRQ buf. */ 1623 (*rxq->mprq_bufs)[rq_idx] = rep; 1624 /* Replace WQE. */ 1625 addr = mlx5_mprq_buf_addr(rep, strd_n); 1626 wqe->addr = rte_cpu_to_be_64((uintptr_t)addr); 1627 /* If there's only one MR, no need to replace LKey in WQE. */ 1628 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) 1629 wqe->lkey = mlx5_rx_addr2mr(rxq, (uintptr_t)addr); 1630 /* Stash a mbuf for next replacement. */ 1631 if (likely(!rte_mempool_get(rxq->mprq_mp, (void **)&rep))) 1632 rxq->mprq_repl = rep; 1633 else 1634 rxq->mprq_repl = NULL; 1635 } 1636 1637 /** 1638 * DPDK callback for RX with Multi-Packet RQ support. 1639 * 1640 * @param dpdk_rxq 1641 * Generic pointer to RX queue structure. 1642 * @param[out] pkts 1643 * Array to store received packets. 1644 * @param pkts_n 1645 * Maximum number of packets in array. 1646 * 1647 * @return 1648 * Number of packets successfully received (<= pkts_n). 1649 */ 1650 uint16_t 1651 mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 1652 { 1653 struct mlx5_rxq_data *rxq = dpdk_rxq; 1654 const unsigned int strd_n = 1 << rxq->strd_num_n; 1655 const unsigned int strd_sz = 1 << rxq->strd_sz_n; 1656 const unsigned int strd_shift = 1657 MLX5_MPRQ_STRIDE_SHIFT_BYTE * rxq->strd_shift_en; 1658 const unsigned int cq_mask = (1 << rxq->cqe_n) - 1; 1659 const unsigned int wq_mask = (1 << rxq->elts_n) - 1; 1660 volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; 1661 unsigned int i = 0; 1662 uint32_t rq_ci = rxq->rq_ci; 1663 uint16_t consumed_strd = rxq->consumed_strd; 1664 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; 1665 1666 while (i < pkts_n) { 1667 struct rte_mbuf *pkt; 1668 void *addr; 1669 int ret; 1670 uint32_t len; 1671 uint16_t strd_cnt; 1672 uint16_t strd_idx; 1673 uint32_t offset; 1674 uint32_t byte_cnt; 1675 int32_t hdrm_overlap; 1676 volatile struct mlx5_mini_cqe8 *mcqe = NULL; 1677 uint32_t rss_hash_res = 0; 1678 1679 if (consumed_strd == strd_n) { 1680 /* Replace WQE only if the buffer is still in use. */ 1681 if (rte_atomic16_read(&buf->refcnt) > 1) { 1682 mprq_buf_replace(rxq, rq_ci & wq_mask, strd_n); 1683 /* Release the old buffer. */ 1684 mlx5_mprq_buf_free(buf); 1685 } else if (unlikely(rxq->mprq_repl == NULL)) { 1686 struct mlx5_mprq_buf *rep; 1687 1688 /* 1689 * Currently, the MPRQ mempool is out of buffer 1690 * and doing memcpy regardless of the size of Rx 1691 * packet. Retry allocation to get back to 1692 * normal. 1693 */ 1694 if (!rte_mempool_get(rxq->mprq_mp, 1695 (void **)&rep)) 1696 rxq->mprq_repl = rep; 1697 } 1698 /* Advance to the next WQE. */ 1699 consumed_strd = 0; 1700 ++rq_ci; 1701 buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; 1702 } 1703 cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; 1704 ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe); 1705 if (!ret) 1706 break; 1707 byte_cnt = ret; 1708 strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >> 1709 MLX5_MPRQ_STRIDE_NUM_SHIFT; 1710 MLX5_ASSERT(strd_cnt); 1711 consumed_strd += strd_cnt; 1712 if (byte_cnt & MLX5_MPRQ_FILLER_MASK) 1713 continue; 1714 if (mcqe == NULL) { 1715 rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res); 1716 strd_idx = rte_be_to_cpu_16(cqe->wqe_counter); 1717 } else { 1718 /* mini-CQE for MPRQ doesn't have hash result. */ 1719 strd_idx = rte_be_to_cpu_16(mcqe->stride_idx); 1720 } 1721 MLX5_ASSERT(strd_idx < strd_n); 1722 MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) & 1723 wq_mask)); 1724 pkt = rte_pktmbuf_alloc(rxq->mp); 1725 if (unlikely(pkt == NULL)) { 1726 ++rxq->stats.rx_nombuf; 1727 break; 1728 } 1729 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT; 1730 MLX5_ASSERT((int)len >= (rxq->crc_present << 2)); 1731 if (rxq->crc_present) 1732 len -= RTE_ETHER_CRC_LEN; 1733 offset = strd_idx * strd_sz + strd_shift; 1734 addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf, strd_n), offset); 1735 hdrm_overlap = len + RTE_PKTMBUF_HEADROOM - strd_cnt * strd_sz; 1736 /* 1737 * Memcpy packets to the target mbuf if: 1738 * - The size of packet is smaller than mprq_max_memcpy_len. 1739 * - Out of buffer in the Mempool for Multi-Packet RQ. 1740 * - The packet's stride overlaps a headroom and scatter is off. 1741 */ 1742 if (len <= rxq->mprq_max_memcpy_len || 1743 rxq->mprq_repl == NULL || 1744 (hdrm_overlap > 0 && !rxq->strd_scatter_en)) { 1745 if (likely(rte_pktmbuf_tailroom(pkt) >= len)) { 1746 rte_memcpy(rte_pktmbuf_mtod(pkt, void *), 1747 addr, len); 1748 DATA_LEN(pkt) = len; 1749 } else if (rxq->strd_scatter_en) { 1750 struct rte_mbuf *prev = pkt; 1751 uint32_t seg_len = 1752 RTE_MIN(rte_pktmbuf_tailroom(pkt), len); 1753 uint32_t rem_len = len - seg_len; 1754 1755 rte_memcpy(rte_pktmbuf_mtod(pkt, void *), 1756 addr, seg_len); 1757 DATA_LEN(pkt) = seg_len; 1758 while (rem_len) { 1759 struct rte_mbuf *next = 1760 rte_pktmbuf_alloc(rxq->mp); 1761 1762 if (unlikely(next == NULL)) { 1763 rte_pktmbuf_free(pkt); 1764 ++rxq->stats.rx_nombuf; 1765 goto out; 1766 } 1767 NEXT(prev) = next; 1768 SET_DATA_OFF(next, 0); 1769 addr = RTE_PTR_ADD(addr, seg_len); 1770 seg_len = RTE_MIN 1771 (rte_pktmbuf_tailroom(next), 1772 rem_len); 1773 rte_memcpy 1774 (rte_pktmbuf_mtod(next, void *), 1775 addr, seg_len); 1776 DATA_LEN(next) = seg_len; 1777 rem_len -= seg_len; 1778 prev = next; 1779 ++NB_SEGS(pkt); 1780 } 1781 } else { 1782 rte_pktmbuf_free_seg(pkt); 1783 ++rxq->stats.idropped; 1784 continue; 1785 } 1786 } else { 1787 rte_iova_t buf_iova; 1788 struct rte_mbuf_ext_shared_info *shinfo; 1789 uint16_t buf_len = strd_cnt * strd_sz; 1790 void *buf_addr; 1791 1792 /* Increment the refcnt of the whole chunk. */ 1793 rte_atomic16_add_return(&buf->refcnt, 1); 1794 MLX5_ASSERT((uint16_t)rte_atomic16_read(&buf->refcnt) <= 1795 strd_n + 1); 1796 buf_addr = RTE_PTR_SUB(addr, RTE_PKTMBUF_HEADROOM); 1797 /* 1798 * MLX5 device doesn't use iova but it is necessary in a 1799 * case where the Rx packet is transmitted via a 1800 * different PMD. 1801 */ 1802 buf_iova = rte_mempool_virt2iova(buf) + 1803 RTE_PTR_DIFF(buf_addr, buf); 1804 shinfo = &buf->shinfos[strd_idx]; 1805 rte_mbuf_ext_refcnt_set(shinfo, 1); 1806 /* 1807 * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when 1808 * attaching the stride to mbuf and more offload flags 1809 * will be added below by calling rxq_cq_to_mbuf(). 1810 * Other fields will be overwritten. 1811 */ 1812 rte_pktmbuf_attach_extbuf(pkt, buf_addr, buf_iova, 1813 buf_len, shinfo); 1814 /* Set mbuf head-room. */ 1815 SET_DATA_OFF(pkt, RTE_PKTMBUF_HEADROOM); 1816 MLX5_ASSERT(pkt->ol_flags == EXT_ATTACHED_MBUF); 1817 MLX5_ASSERT(rte_pktmbuf_tailroom(pkt) >= 1818 len - (hdrm_overlap > 0 ? hdrm_overlap : 0)); 1819 DATA_LEN(pkt) = len; 1820 /* 1821 * Copy the last fragment of a packet (up to headroom 1822 * size bytes) in case there is a stride overlap with 1823 * a next packet's headroom. Allocate a separate mbuf 1824 * to store this fragment and link it. Scatter is on. 1825 */ 1826 if (hdrm_overlap > 0) { 1827 MLX5_ASSERT(rxq->strd_scatter_en); 1828 struct rte_mbuf *seg = 1829 rte_pktmbuf_alloc(rxq->mp); 1830 1831 if (unlikely(seg == NULL)) { 1832 rte_pktmbuf_free_seg(pkt); 1833 ++rxq->stats.rx_nombuf; 1834 break; 1835 } 1836 SET_DATA_OFF(seg, 0); 1837 rte_memcpy(rte_pktmbuf_mtod(seg, void *), 1838 RTE_PTR_ADD(addr, len - hdrm_overlap), 1839 hdrm_overlap); 1840 DATA_LEN(seg) = hdrm_overlap; 1841 DATA_LEN(pkt) = len - hdrm_overlap; 1842 NEXT(pkt) = seg; 1843 NB_SEGS(pkt) = 2; 1844 } 1845 } 1846 rxq_cq_to_mbuf(rxq, pkt, cqe, rss_hash_res); 1847 if (cqe->lro_num_seg > 1) { 1848 mlx5_lro_update_hdr(addr, cqe, len); 1849 pkt->ol_flags |= PKT_RX_LRO; 1850 pkt->tso_segsz = len / cqe->lro_num_seg; 1851 } 1852 PKT_LEN(pkt) = len; 1853 PORT(pkt) = rxq->port_id; 1854 #ifdef MLX5_PMD_SOFT_COUNTERS 1855 /* Increment bytes counter. */ 1856 rxq->stats.ibytes += PKT_LEN(pkt); 1857 #endif 1858 /* Return packet. */ 1859 *(pkts++) = pkt; 1860 ++i; 1861 } 1862 out: 1863 /* Update the consumer indexes. */ 1864 rxq->consumed_strd = consumed_strd; 1865 rte_cio_wmb(); 1866 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1867 if (rq_ci != rxq->rq_ci) { 1868 rxq->rq_ci = rq_ci; 1869 rte_cio_wmb(); 1870 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 1871 } 1872 #ifdef MLX5_PMD_SOFT_COUNTERS 1873 /* Increment packets counter. */ 1874 rxq->stats.ipackets += i; 1875 #endif 1876 return i; 1877 } 1878 1879 /** 1880 * Dummy DPDK callback for TX. 1881 * 1882 * This function is used to temporarily replace the real callback during 1883 * unsafe control operations on the queue, or in case of error. 1884 * 1885 * @param dpdk_txq 1886 * Generic pointer to TX queue structure. 1887 * @param[in] pkts 1888 * Packets to transmit. 1889 * @param pkts_n 1890 * Number of packets in array. 1891 * 1892 * @return 1893 * Number of packets successfully transmitted (<= pkts_n). 1894 */ 1895 uint16_t 1896 removed_tx_burst(void *dpdk_txq __rte_unused, 1897 struct rte_mbuf **pkts __rte_unused, 1898 uint16_t pkts_n __rte_unused) 1899 { 1900 rte_mb(); 1901 return 0; 1902 } 1903 1904 /** 1905 * Dummy DPDK callback for RX. 1906 * 1907 * This function is used to temporarily replace the real callback during 1908 * unsafe control operations on the queue, or in case of error. 1909 * 1910 * @param dpdk_rxq 1911 * Generic pointer to RX queue structure. 1912 * @param[out] pkts 1913 * Array to store received packets. 1914 * @param pkts_n 1915 * Maximum number of packets in array. 1916 * 1917 * @return 1918 * Number of packets successfully received (<= pkts_n). 1919 */ 1920 uint16_t 1921 removed_rx_burst(void *dpdk_txq __rte_unused, 1922 struct rte_mbuf **pkts __rte_unused, 1923 uint16_t pkts_n __rte_unused) 1924 { 1925 rte_mb(); 1926 return 0; 1927 } 1928 1929 /* 1930 * Vectorized Rx/Tx routines are not compiled in when required vector 1931 * instructions are not supported on a target architecture. The following null 1932 * stubs are needed for linkage when those are not included outside of this file 1933 * (e.g. mlx5_rxtx_vec_sse.c for x86). 1934 */ 1935 1936 __rte_weak uint16_t 1937 mlx5_rx_burst_vec(void *dpdk_txq __rte_unused, 1938 struct rte_mbuf **pkts __rte_unused, 1939 uint16_t pkts_n __rte_unused) 1940 { 1941 return 0; 1942 } 1943 1944 __rte_weak int 1945 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) 1946 { 1947 return -ENOTSUP; 1948 } 1949 1950 __rte_weak int 1951 mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused) 1952 { 1953 return -ENOTSUP; 1954 } 1955 1956 /** 1957 * Free the mbufs from the linear array of pointers. 1958 * 1959 * @param pkts 1960 * Pointer to array of packets to be free. 1961 * @param pkts_n 1962 * Number of packets to be freed. 1963 * @param olx 1964 * Configured Tx offloads mask. It is fully defined at 1965 * compile time and may be used for optimization. 1966 */ 1967 static __rte_always_inline void 1968 mlx5_tx_free_mbuf(struct rte_mbuf **restrict pkts, 1969 unsigned int pkts_n, 1970 unsigned int olx __rte_unused) 1971 { 1972 struct rte_mempool *pool = NULL; 1973 struct rte_mbuf **p_free = NULL; 1974 struct rte_mbuf *mbuf; 1975 unsigned int n_free = 0; 1976 1977 /* 1978 * The implemented algorithm eliminates 1979 * copying pointers to temporary array 1980 * for rte_mempool_put_bulk() calls. 1981 */ 1982 MLX5_ASSERT(pkts); 1983 MLX5_ASSERT(pkts_n); 1984 for (;;) { 1985 for (;;) { 1986 /* 1987 * Decrement mbuf reference counter, detach 1988 * indirect and external buffers if needed. 1989 */ 1990 mbuf = rte_pktmbuf_prefree_seg(*pkts); 1991 if (likely(mbuf != NULL)) { 1992 MLX5_ASSERT(mbuf == *pkts); 1993 if (likely(n_free != 0)) { 1994 if (unlikely(pool != mbuf->pool)) 1995 /* From different pool. */ 1996 break; 1997 } else { 1998 /* Start new scan array. */ 1999 pool = mbuf->pool; 2000 p_free = pkts; 2001 } 2002 ++n_free; 2003 ++pkts; 2004 --pkts_n; 2005 if (unlikely(pkts_n == 0)) { 2006 mbuf = NULL; 2007 break; 2008 } 2009 } else { 2010 /* 2011 * This happens if mbuf is still referenced. 2012 * We can't put it back to the pool, skip. 2013 */ 2014 ++pkts; 2015 --pkts_n; 2016 if (unlikely(n_free != 0)) 2017 /* There is some array to free.*/ 2018 break; 2019 if (unlikely(pkts_n == 0)) 2020 /* Last mbuf, nothing to free. */ 2021 return; 2022 } 2023 } 2024 for (;;) { 2025 /* 2026 * This loop is implemented to avoid multiple 2027 * inlining of rte_mempool_put_bulk(). 2028 */ 2029 MLX5_ASSERT(pool); 2030 MLX5_ASSERT(p_free); 2031 MLX5_ASSERT(n_free); 2032 /* 2033 * Free the array of pre-freed mbufs 2034 * belonging to the same memory pool. 2035 */ 2036 rte_mempool_put_bulk(pool, (void *)p_free, n_free); 2037 if (unlikely(mbuf != NULL)) { 2038 /* There is the request to start new scan. */ 2039 pool = mbuf->pool; 2040 p_free = pkts++; 2041 n_free = 1; 2042 --pkts_n; 2043 if (likely(pkts_n != 0)) 2044 break; 2045 /* 2046 * This is the last mbuf to be freed. 2047 * Do one more loop iteration to complete. 2048 * This is rare case of the last unique mbuf. 2049 */ 2050 mbuf = NULL; 2051 continue; 2052 } 2053 if (likely(pkts_n == 0)) 2054 return; 2055 n_free = 0; 2056 break; 2057 } 2058 } 2059 } 2060 2061 /** 2062 * Free the mbuf from the elts ring buffer till new tail. 2063 * 2064 * @param txq 2065 * Pointer to Tx queue structure. 2066 * @param tail 2067 * Index in elts to free up to, becomes new elts tail. 2068 * @param olx 2069 * Configured Tx offloads mask. It is fully defined at 2070 * compile time and may be used for optimization. 2071 */ 2072 static __rte_always_inline void 2073 mlx5_tx_free_elts(struct mlx5_txq_data *restrict txq, 2074 uint16_t tail, 2075 unsigned int olx __rte_unused) 2076 { 2077 uint16_t n_elts = tail - txq->elts_tail; 2078 2079 MLX5_ASSERT(n_elts); 2080 MLX5_ASSERT(n_elts <= txq->elts_s); 2081 /* 2082 * Implement a loop to support ring buffer wraparound 2083 * with single inlining of mlx5_tx_free_mbuf(). 2084 */ 2085 do { 2086 unsigned int part; 2087 2088 part = txq->elts_s - (txq->elts_tail & txq->elts_m); 2089 part = RTE_MIN(part, n_elts); 2090 MLX5_ASSERT(part); 2091 MLX5_ASSERT(part <= txq->elts_s); 2092 mlx5_tx_free_mbuf(&txq->elts[txq->elts_tail & txq->elts_m], 2093 part, olx); 2094 txq->elts_tail += part; 2095 n_elts -= part; 2096 } while (n_elts); 2097 } 2098 2099 /** 2100 * Store the mbuf being sent into elts ring buffer. 2101 * On Tx completion these mbufs will be freed. 2102 * 2103 * @param txq 2104 * Pointer to Tx queue structure. 2105 * @param pkts 2106 * Pointer to array of packets to be stored. 2107 * @param pkts_n 2108 * Number of packets to be stored. 2109 * @param olx 2110 * Configured Tx offloads mask. It is fully defined at 2111 * compile time and may be used for optimization. 2112 */ 2113 static __rte_always_inline void 2114 mlx5_tx_copy_elts(struct mlx5_txq_data *restrict txq, 2115 struct rte_mbuf **restrict pkts, 2116 unsigned int pkts_n, 2117 unsigned int olx __rte_unused) 2118 { 2119 unsigned int part; 2120 struct rte_mbuf **elts = (struct rte_mbuf **)txq->elts; 2121 2122 MLX5_ASSERT(pkts); 2123 MLX5_ASSERT(pkts_n); 2124 part = txq->elts_s - (txq->elts_head & txq->elts_m); 2125 MLX5_ASSERT(part); 2126 MLX5_ASSERT(part <= txq->elts_s); 2127 /* This code is a good candidate for vectorizing with SIMD. */ 2128 rte_memcpy((void *)(elts + (txq->elts_head & txq->elts_m)), 2129 (void *)pkts, 2130 RTE_MIN(part, pkts_n) * sizeof(struct rte_mbuf *)); 2131 txq->elts_head += pkts_n; 2132 if (unlikely(part < pkts_n)) 2133 /* The copy is wrapping around the elts array. */ 2134 rte_memcpy((void *)elts, (void *)(pkts + part), 2135 (pkts_n - part) * sizeof(struct rte_mbuf *)); 2136 } 2137 2138 /** 2139 * Update completion queue consuming index via doorbell 2140 * and flush the completed data buffers. 2141 * 2142 * @param txq 2143 * Pointer to TX queue structure. 2144 * @param valid CQE pointer 2145 * if not NULL update txq->wqe_pi and flush the buffers 2146 * @param olx 2147 * Configured Tx offloads mask. It is fully defined at 2148 * compile time and may be used for optimization. 2149 */ 2150 static __rte_always_inline void 2151 mlx5_tx_comp_flush(struct mlx5_txq_data *restrict txq, 2152 volatile struct mlx5_cqe *last_cqe, 2153 unsigned int olx __rte_unused) 2154 { 2155 if (likely(last_cqe != NULL)) { 2156 uint16_t tail; 2157 2158 txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter); 2159 tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m]; 2160 if (likely(tail != txq->elts_tail)) { 2161 mlx5_tx_free_elts(txq, tail, olx); 2162 MLX5_ASSERT(tail == txq->elts_tail); 2163 } 2164 } 2165 } 2166 2167 /** 2168 * Manage TX completions. This routine checks the CQ for 2169 * arrived CQEs, deduces the last accomplished WQE in SQ, 2170 * updates SQ producing index and frees all completed mbufs. 2171 * 2172 * @param txq 2173 * Pointer to TX queue structure. 2174 * @param olx 2175 * Configured Tx offloads mask. It is fully defined at 2176 * compile time and may be used for optimization. 2177 * 2178 * NOTE: not inlined intentionally, it makes tx_burst 2179 * routine smaller, simple and faster - from experiments. 2180 */ 2181 static void 2182 mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq, 2183 unsigned int olx __rte_unused) 2184 { 2185 unsigned int count = MLX5_TX_COMP_MAX_CQE; 2186 volatile struct mlx5_cqe *last_cqe = NULL; 2187 bool ring_doorbell = false; 2188 int ret; 2189 2190 static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value"); 2191 static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value"); 2192 do { 2193 volatile struct mlx5_cqe *cqe; 2194 2195 cqe = &txq->cqes[txq->cq_ci & txq->cqe_m]; 2196 ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci); 2197 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 2198 if (likely(ret != MLX5_CQE_STATUS_ERR)) { 2199 /* No new CQEs in completion queue. */ 2200 MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN); 2201 break; 2202 } 2203 /* 2204 * Some error occurred, try to restart. 2205 * We have no barrier after WQE related Doorbell 2206 * written, make sure all writes are completed 2207 * here, before we might perform SQ reset. 2208 */ 2209 rte_wmb(); 2210 ret = mlx5_tx_error_cqe_handle 2211 (txq, (volatile struct mlx5_err_cqe *)cqe); 2212 if (unlikely(ret < 0)) { 2213 /* 2214 * Some error occurred on queue error 2215 * handling, we do not advance the index 2216 * here, allowing to retry on next call. 2217 */ 2218 return; 2219 } 2220 /* 2221 * We are going to fetch all entries with 2222 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status. 2223 * The send queue is supposed to be empty. 2224 */ 2225 ring_doorbell = true; 2226 ++txq->cq_ci; 2227 txq->cq_pi = txq->cq_ci; 2228 last_cqe = NULL; 2229 continue; 2230 } 2231 /* Normal transmit completion. */ 2232 MLX5_ASSERT(txq->cq_ci != txq->cq_pi); 2233 MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) == 2234 cqe->wqe_counter); 2235 ring_doorbell = true; 2236 ++txq->cq_ci; 2237 last_cqe = cqe; 2238 /* 2239 * We have to restrict the amount of processed CQEs 2240 * in one tx_burst routine call. The CQ may be large 2241 * and many CQEs may be updated by the NIC in one 2242 * transaction. Buffers freeing is time consuming, 2243 * multiple iterations may introduce significant 2244 * latency. 2245 */ 2246 if (likely(--count == 0)) 2247 break; 2248 } while (true); 2249 if (likely(ring_doorbell)) { 2250 /* Ring doorbell to notify hardware. */ 2251 rte_compiler_barrier(); 2252 *txq->cq_db = rte_cpu_to_be_32(txq->cq_ci); 2253 mlx5_tx_comp_flush(txq, last_cqe, olx); 2254 } 2255 } 2256 2257 /** 2258 * Check if the completion request flag should be set in the last WQE. 2259 * Both pushed mbufs and WQEs are monitored and the completion request 2260 * flag is set if any of thresholds is reached. 2261 * 2262 * @param txq 2263 * Pointer to TX queue structure. 2264 * @param loc 2265 * Pointer to burst routine local context. 2266 * @param olx 2267 * Configured Tx offloads mask. It is fully defined at 2268 * compile time and may be used for optimization. 2269 */ 2270 static __rte_always_inline void 2271 mlx5_tx_request_completion(struct mlx5_txq_data *restrict txq, 2272 struct mlx5_txq_local *restrict loc, 2273 unsigned int olx) 2274 { 2275 uint16_t head = txq->elts_head; 2276 unsigned int part; 2277 2278 part = MLX5_TXOFF_CONFIG(INLINE) ? 2279 0 : loc->pkts_sent - loc->pkts_copy; 2280 head += part; 2281 if ((uint16_t)(head - txq->elts_comp) >= MLX5_TX_COMP_THRESH || 2282 (MLX5_TXOFF_CONFIG(INLINE) && 2283 (uint16_t)(txq->wqe_ci - txq->wqe_comp) >= txq->wqe_thres)) { 2284 volatile struct mlx5_wqe *last = loc->wqe_last; 2285 2286 MLX5_ASSERT(last); 2287 txq->elts_comp = head; 2288 if (MLX5_TXOFF_CONFIG(INLINE)) 2289 txq->wqe_comp = txq->wqe_ci; 2290 /* Request unconditional completion on last WQE. */ 2291 last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << 2292 MLX5_COMP_MODE_OFFSET); 2293 /* Save elts_head in dedicated free on completion queue. */ 2294 #ifdef RTE_LIBRTE_MLX5_DEBUG 2295 txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head | 2296 (last->cseg.opcode >> 8) << 16; 2297 #else 2298 txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head; 2299 #endif 2300 /* A CQE slot must always be available. */ 2301 MLX5_ASSERT((txq->cq_pi - txq->cq_ci) <= txq->cqe_s); 2302 } 2303 } 2304 2305 /** 2306 * DPDK callback to check the status of a tx descriptor. 2307 * 2308 * @param tx_queue 2309 * The tx queue. 2310 * @param[in] offset 2311 * The index of the descriptor in the ring. 2312 * 2313 * @return 2314 * The status of the tx descriptor. 2315 */ 2316 int 2317 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset) 2318 { 2319 struct mlx5_txq_data *restrict txq = tx_queue; 2320 uint16_t used; 2321 2322 mlx5_tx_handle_completion(txq, 0); 2323 used = txq->elts_head - txq->elts_tail; 2324 if (offset < used) 2325 return RTE_ETH_TX_DESC_FULL; 2326 return RTE_ETH_TX_DESC_DONE; 2327 } 2328 2329 /** 2330 * Build the Control Segment with specified opcode: 2331 * - MLX5_OPCODE_SEND 2332 * - MLX5_OPCODE_ENHANCED_MPSW 2333 * - MLX5_OPCODE_TSO 2334 * 2335 * @param txq 2336 * Pointer to TX queue structure. 2337 * @param loc 2338 * Pointer to burst routine local context. 2339 * @param wqe 2340 * Pointer to WQE to fill with built Control Segment. 2341 * @param ds 2342 * Supposed length of WQE in segments. 2343 * @param opcode 2344 * SQ WQE opcode to put into Control Segment. 2345 * @param olx 2346 * Configured Tx offloads mask. It is fully defined at 2347 * compile time and may be used for optimization. 2348 */ 2349 static __rte_always_inline void 2350 mlx5_tx_cseg_init(struct mlx5_txq_data *restrict txq, 2351 struct mlx5_txq_local *restrict loc __rte_unused, 2352 struct mlx5_wqe *restrict wqe, 2353 unsigned int ds, 2354 unsigned int opcode, 2355 unsigned int olx __rte_unused) 2356 { 2357 struct mlx5_wqe_cseg *restrict cs = &wqe->cseg; 2358 2359 /* For legacy MPW replace the EMPW by TSO with modifier. */ 2360 if (MLX5_TXOFF_CONFIG(MPW) && opcode == MLX5_OPCODE_ENHANCED_MPSW) 2361 opcode = MLX5_OPCODE_TSO | MLX5_OPC_MOD_MPW << 24; 2362 cs->opcode = rte_cpu_to_be_32((txq->wqe_ci << 8) | opcode); 2363 cs->sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 2364 cs->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR << 2365 MLX5_COMP_MODE_OFFSET); 2366 cs->misc = RTE_BE32(0); 2367 } 2368 2369 /** 2370 * Build the Ethernet Segment without inlined data. 2371 * Supports Software Parser, Checksums and VLAN 2372 * insertion Tx offload features. 2373 * 2374 * @param txq 2375 * Pointer to TX queue structure. 2376 * @param loc 2377 * Pointer to burst routine local context. 2378 * @param wqe 2379 * Pointer to WQE to fill with built Ethernet Segment. 2380 * @param olx 2381 * Configured Tx offloads mask. It is fully defined at 2382 * compile time and may be used for optimization. 2383 */ 2384 static __rte_always_inline void 2385 mlx5_tx_eseg_none(struct mlx5_txq_data *restrict txq __rte_unused, 2386 struct mlx5_txq_local *restrict loc, 2387 struct mlx5_wqe *restrict wqe, 2388 unsigned int olx) 2389 { 2390 struct mlx5_wqe_eseg *restrict es = &wqe->eseg; 2391 uint32_t csum; 2392 2393 /* 2394 * Calculate and set check sum flags first, dword field 2395 * in segment may be shared with Software Parser flags. 2396 */ 2397 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2398 es->flags = rte_cpu_to_le_32(csum); 2399 /* 2400 * Calculate and set Software Parser offsets and flags. 2401 * These flags a set for custom UDP and IP tunnel packets. 2402 */ 2403 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2404 /* Fill metadata field if needed. */ 2405 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2406 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2407 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2408 /* Engage VLAN tag insertion feature if requested. */ 2409 if (MLX5_TXOFF_CONFIG(VLAN) && 2410 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 2411 /* 2412 * We should get here only if device support 2413 * this feature correctly. 2414 */ 2415 MLX5_ASSERT(txq->vlan_en); 2416 es->inline_hdr = rte_cpu_to_be_32(MLX5_ETH_WQE_VLAN_INSERT | 2417 loc->mbuf->vlan_tci); 2418 } else { 2419 es->inline_hdr = RTE_BE32(0); 2420 } 2421 } 2422 2423 /** 2424 * Build the Ethernet Segment with minimal inlined data 2425 * of MLX5_ESEG_MIN_INLINE_SIZE bytes length. This is 2426 * used to fill the gap in single WQEBB WQEs. 2427 * Supports Software Parser, Checksums and VLAN 2428 * insertion Tx offload features. 2429 * 2430 * @param txq 2431 * Pointer to TX queue structure. 2432 * @param loc 2433 * Pointer to burst routine local context. 2434 * @param wqe 2435 * Pointer to WQE to fill with built Ethernet Segment. 2436 * @param vlan 2437 * Length of VLAN tag insertion if any. 2438 * @param olx 2439 * Configured Tx offloads mask. It is fully defined at 2440 * compile time and may be used for optimization. 2441 */ 2442 static __rte_always_inline void 2443 mlx5_tx_eseg_dmin(struct mlx5_txq_data *restrict txq __rte_unused, 2444 struct mlx5_txq_local *restrict loc, 2445 struct mlx5_wqe *restrict wqe, 2446 unsigned int vlan, 2447 unsigned int olx) 2448 { 2449 struct mlx5_wqe_eseg *restrict es = &wqe->eseg; 2450 uint32_t csum; 2451 uint8_t *psrc, *pdst; 2452 2453 /* 2454 * Calculate and set check sum flags first, dword field 2455 * in segment may be shared with Software Parser flags. 2456 */ 2457 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2458 es->flags = rte_cpu_to_le_32(csum); 2459 /* 2460 * Calculate and set Software Parser offsets and flags. 2461 * These flags a set for custom UDP and IP tunnel packets. 2462 */ 2463 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2464 /* Fill metadata field if needed. */ 2465 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2466 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2467 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2468 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 2469 (sizeof(uint16_t) + 2470 sizeof(rte_v128u32_t)), 2471 "invalid Ethernet Segment data size"); 2472 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 2473 (sizeof(uint16_t) + 2474 sizeof(struct rte_vlan_hdr) + 2475 2 * RTE_ETHER_ADDR_LEN), 2476 "invalid Ethernet Segment data size"); 2477 psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *); 2478 es->inline_hdr_sz = RTE_BE16(MLX5_ESEG_MIN_INLINE_SIZE); 2479 es->inline_data = *(unaligned_uint16_t *)psrc; 2480 psrc += sizeof(uint16_t); 2481 pdst = (uint8_t *)(es + 1); 2482 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) { 2483 /* Implement VLAN tag insertion as part inline data. */ 2484 memcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t)); 2485 pdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2486 psrc += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2487 /* Insert VLAN ethertype + VLAN tag. */ 2488 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32 2489 ((RTE_ETHER_TYPE_VLAN << 16) | 2490 loc->mbuf->vlan_tci); 2491 pdst += sizeof(struct rte_vlan_hdr); 2492 /* Copy the rest two bytes from packet data. */ 2493 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, sizeof(uint16_t))); 2494 *(uint16_t *)pdst = *(unaligned_uint16_t *)psrc; 2495 } else { 2496 /* Fill the gap in the title WQEBB with inline data. */ 2497 rte_mov16(pdst, psrc); 2498 } 2499 } 2500 2501 /** 2502 * Build the Ethernet Segment with entire packet 2503 * data inlining. Checks the boundary of WQEBB and 2504 * ring buffer wrapping, supports Software Parser, 2505 * Checksums and VLAN insertion Tx offload features. 2506 * 2507 * @param txq 2508 * Pointer to TX queue structure. 2509 * @param loc 2510 * Pointer to burst routine local context. 2511 * @param wqe 2512 * Pointer to WQE to fill with built Ethernet Segment. 2513 * @param vlan 2514 * Length of VLAN tag insertion if any. 2515 * @param inlen 2516 * Length of data to inline (VLAN included, if any). 2517 * @param tso 2518 * TSO flag, set mss field from the packet. 2519 * @param olx 2520 * Configured Tx offloads mask. It is fully defined at 2521 * compile time and may be used for optimization. 2522 * 2523 * @return 2524 * Pointer to the next Data Segment (aligned and wrapped around). 2525 */ 2526 static __rte_always_inline struct mlx5_wqe_dseg * 2527 mlx5_tx_eseg_data(struct mlx5_txq_data *restrict txq, 2528 struct mlx5_txq_local *restrict loc, 2529 struct mlx5_wqe *restrict wqe, 2530 unsigned int vlan, 2531 unsigned int inlen, 2532 unsigned int tso, 2533 unsigned int olx) 2534 { 2535 struct mlx5_wqe_eseg *restrict es = &wqe->eseg; 2536 uint32_t csum; 2537 uint8_t *psrc, *pdst; 2538 unsigned int part; 2539 2540 /* 2541 * Calculate and set check sum flags first, dword field 2542 * in segment may be shared with Software Parser flags. 2543 */ 2544 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2545 if (tso) { 2546 csum <<= 24; 2547 csum |= loc->mbuf->tso_segsz; 2548 es->flags = rte_cpu_to_be_32(csum); 2549 } else { 2550 es->flags = rte_cpu_to_le_32(csum); 2551 } 2552 /* 2553 * Calculate and set Software Parser offsets and flags. 2554 * These flags a set for custom UDP and IP tunnel packets. 2555 */ 2556 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2557 /* Fill metadata field if needed. */ 2558 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2559 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2560 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2561 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 2562 (sizeof(uint16_t) + 2563 sizeof(rte_v128u32_t)), 2564 "invalid Ethernet Segment data size"); 2565 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 2566 (sizeof(uint16_t) + 2567 sizeof(struct rte_vlan_hdr) + 2568 2 * RTE_ETHER_ADDR_LEN), 2569 "invalid Ethernet Segment data size"); 2570 psrc = rte_pktmbuf_mtod(loc->mbuf, uint8_t *); 2571 es->inline_hdr_sz = rte_cpu_to_be_16(inlen); 2572 es->inline_data = *(unaligned_uint16_t *)psrc; 2573 psrc += sizeof(uint16_t); 2574 pdst = (uint8_t *)(es + 1); 2575 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) { 2576 /* Implement VLAN tag insertion as part inline data. */ 2577 memcpy(pdst, psrc, 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t)); 2578 pdst += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2579 psrc += 2 * RTE_ETHER_ADDR_LEN - sizeof(uint16_t); 2580 /* Insert VLAN ethertype + VLAN tag. */ 2581 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32 2582 ((RTE_ETHER_TYPE_VLAN << 16) | 2583 loc->mbuf->vlan_tci); 2584 pdst += sizeof(struct rte_vlan_hdr); 2585 /* Copy the rest two bytes from packet data. */ 2586 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, sizeof(uint16_t))); 2587 *(uint16_t *)pdst = *(unaligned_uint16_t *)psrc; 2588 psrc += sizeof(uint16_t); 2589 } else { 2590 /* Fill the gap in the title WQEBB with inline data. */ 2591 rte_mov16(pdst, psrc); 2592 psrc += sizeof(rte_v128u32_t); 2593 } 2594 pdst = (uint8_t *)(es + 2); 2595 MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE); 2596 MLX5_ASSERT(pdst < (uint8_t *)txq->wqes_end); 2597 inlen -= MLX5_ESEG_MIN_INLINE_SIZE; 2598 if (!inlen) { 2599 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE)); 2600 return (struct mlx5_wqe_dseg *)pdst; 2601 } 2602 /* 2603 * The WQEBB space availability is checked by caller. 2604 * Here we should be aware of WQE ring buffer wraparound only. 2605 */ 2606 part = (uint8_t *)txq->wqes_end - pdst; 2607 part = RTE_MIN(part, inlen); 2608 do { 2609 rte_memcpy(pdst, psrc, part); 2610 inlen -= part; 2611 if (likely(!inlen)) { 2612 /* 2613 * If return value is not used by the caller 2614 * the code below will be optimized out. 2615 */ 2616 pdst += part; 2617 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 2618 if (unlikely(pdst >= (uint8_t *)txq->wqes_end)) 2619 pdst = (uint8_t *)txq->wqes; 2620 return (struct mlx5_wqe_dseg *)pdst; 2621 } 2622 pdst = (uint8_t *)txq->wqes; 2623 psrc += part; 2624 part = inlen; 2625 } while (true); 2626 } 2627 2628 /** 2629 * Copy data from chain of mbuf to the specified linear buffer. 2630 * Checksums and VLAN insertion Tx offload features. If data 2631 * from some mbuf copied completely this mbuf is freed. Local 2632 * structure is used to keep the byte stream state. 2633 * 2634 * @param pdst 2635 * Pointer to the destination linear buffer. 2636 * @param loc 2637 * Pointer to burst routine local context. 2638 * @param len 2639 * Length of data to be copied. 2640 * @param must 2641 * Length of data to be copied ignoring no inline hint. 2642 * @param olx 2643 * Configured Tx offloads mask. It is fully defined at 2644 * compile time and may be used for optimization. 2645 * 2646 * @return 2647 * Number of actual copied data bytes. This is always greater than or 2648 * equal to must parameter and might be lesser than len in no inline 2649 * hint flag is encountered. 2650 */ 2651 static __rte_always_inline unsigned int 2652 mlx5_tx_mseg_memcpy(uint8_t *pdst, 2653 struct mlx5_txq_local *restrict loc, 2654 unsigned int len, 2655 unsigned int must, 2656 unsigned int olx __rte_unused) 2657 { 2658 struct rte_mbuf *mbuf; 2659 unsigned int part, dlen, copy = 0; 2660 uint8_t *psrc; 2661 2662 MLX5_ASSERT(len); 2663 MLX5_ASSERT(must <= len); 2664 do { 2665 /* Allow zero length packets, must check first. */ 2666 dlen = rte_pktmbuf_data_len(loc->mbuf); 2667 if (dlen <= loc->mbuf_off) { 2668 /* Exhausted packet, just free. */ 2669 mbuf = loc->mbuf; 2670 loc->mbuf = mbuf->next; 2671 rte_pktmbuf_free_seg(mbuf); 2672 loc->mbuf_off = 0; 2673 MLX5_ASSERT(loc->mbuf_nseg > 1); 2674 MLX5_ASSERT(loc->mbuf); 2675 --loc->mbuf_nseg; 2676 if (loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) { 2677 unsigned int diff; 2678 2679 if (copy >= must) { 2680 /* 2681 * We already copied the minimal 2682 * requested amount of data. 2683 */ 2684 return copy; 2685 } 2686 diff = must - copy; 2687 if (diff <= rte_pktmbuf_data_len(loc->mbuf)) { 2688 /* 2689 * Copy only the minimal required 2690 * part of the data buffer. 2691 */ 2692 len = diff; 2693 } 2694 } 2695 continue; 2696 } 2697 dlen -= loc->mbuf_off; 2698 psrc = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *, 2699 loc->mbuf_off); 2700 part = RTE_MIN(len, dlen); 2701 rte_memcpy(pdst, psrc, part); 2702 copy += part; 2703 loc->mbuf_off += part; 2704 len -= part; 2705 if (!len) { 2706 if (loc->mbuf_off >= rte_pktmbuf_data_len(loc->mbuf)) { 2707 loc->mbuf_off = 0; 2708 /* Exhausted packet, just free. */ 2709 mbuf = loc->mbuf; 2710 loc->mbuf = mbuf->next; 2711 rte_pktmbuf_free_seg(mbuf); 2712 loc->mbuf_off = 0; 2713 MLX5_ASSERT(loc->mbuf_nseg >= 1); 2714 --loc->mbuf_nseg; 2715 } 2716 return copy; 2717 } 2718 pdst += part; 2719 } while (true); 2720 } 2721 2722 /** 2723 * Build the Ethernet Segment with inlined data from 2724 * multi-segment packet. Checks the boundary of WQEBB 2725 * and ring buffer wrapping, supports Software Parser, 2726 * Checksums and VLAN insertion Tx offload features. 2727 * 2728 * @param txq 2729 * Pointer to TX queue structure. 2730 * @param loc 2731 * Pointer to burst routine local context. 2732 * @param wqe 2733 * Pointer to WQE to fill with built Ethernet Segment. 2734 * @param vlan 2735 * Length of VLAN tag insertion if any. 2736 * @param inlen 2737 * Length of data to inline (VLAN included, if any). 2738 * @param tso 2739 * TSO flag, set mss field from the packet. 2740 * @param olx 2741 * Configured Tx offloads mask. It is fully defined at 2742 * compile time and may be used for optimization. 2743 * 2744 * @return 2745 * Pointer to the next Data Segment (aligned and 2746 * possible NOT wrapped around - caller should do 2747 * wrapping check on its own). 2748 */ 2749 static __rte_always_inline struct mlx5_wqe_dseg * 2750 mlx5_tx_eseg_mdat(struct mlx5_txq_data *restrict txq, 2751 struct mlx5_txq_local *restrict loc, 2752 struct mlx5_wqe *restrict wqe, 2753 unsigned int vlan, 2754 unsigned int inlen, 2755 unsigned int tso, 2756 unsigned int olx) 2757 { 2758 struct mlx5_wqe_eseg *restrict es = &wqe->eseg; 2759 uint32_t csum; 2760 uint8_t *pdst; 2761 unsigned int part, tlen = 0; 2762 2763 /* 2764 * Calculate and set check sum flags first, uint32_t field 2765 * in segment may be shared with Software Parser flags. 2766 */ 2767 csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0; 2768 if (tso) { 2769 csum <<= 24; 2770 csum |= loc->mbuf->tso_segsz; 2771 es->flags = rte_cpu_to_be_32(csum); 2772 } else { 2773 es->flags = rte_cpu_to_le_32(csum); 2774 } 2775 /* 2776 * Calculate and set Software Parser offsets and flags. 2777 * These flags a set for custom UDP and IP tunnel packets. 2778 */ 2779 es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx); 2780 /* Fill metadata field if needed. */ 2781 es->metadata = MLX5_TXOFF_CONFIG(METADATA) ? 2782 loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 2783 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0; 2784 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 2785 (sizeof(uint16_t) + 2786 sizeof(rte_v128u32_t)), 2787 "invalid Ethernet Segment data size"); 2788 static_assert(MLX5_ESEG_MIN_INLINE_SIZE == 2789 (sizeof(uint16_t) + 2790 sizeof(struct rte_vlan_hdr) + 2791 2 * RTE_ETHER_ADDR_LEN), 2792 "invalid Ethernet Segment data size"); 2793 MLX5_ASSERT(inlen >= MLX5_ESEG_MIN_INLINE_SIZE); 2794 pdst = (uint8_t *)&es->inline_data; 2795 if (MLX5_TXOFF_CONFIG(VLAN) && vlan) { 2796 /* Implement VLAN tag insertion as part inline data. */ 2797 mlx5_tx_mseg_memcpy(pdst, loc, 2798 2 * RTE_ETHER_ADDR_LEN, 2799 2 * RTE_ETHER_ADDR_LEN, olx); 2800 pdst += 2 * RTE_ETHER_ADDR_LEN; 2801 *(unaligned_uint32_t *)pdst = rte_cpu_to_be_32 2802 ((RTE_ETHER_TYPE_VLAN << 16) | 2803 loc->mbuf->vlan_tci); 2804 pdst += sizeof(struct rte_vlan_hdr); 2805 tlen += 2 * RTE_ETHER_ADDR_LEN + sizeof(struct rte_vlan_hdr); 2806 } 2807 MLX5_ASSERT(pdst < (uint8_t *)txq->wqes_end); 2808 /* 2809 * The WQEBB space availability is checked by caller. 2810 * Here we should be aware of WQE ring buffer wraparound only. 2811 */ 2812 part = (uint8_t *)txq->wqes_end - pdst; 2813 part = RTE_MIN(part, inlen - tlen); 2814 MLX5_ASSERT(part); 2815 do { 2816 unsigned int copy; 2817 2818 /* 2819 * Copying may be interrupted inside the routine 2820 * if run into no inline hint flag. 2821 */ 2822 copy = tlen >= txq->inlen_mode ? 0 : (txq->inlen_mode - tlen); 2823 copy = mlx5_tx_mseg_memcpy(pdst, loc, part, copy, olx); 2824 tlen += copy; 2825 if (likely(inlen <= tlen) || copy < part) { 2826 es->inline_hdr_sz = rte_cpu_to_be_16(tlen); 2827 pdst += copy; 2828 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 2829 return (struct mlx5_wqe_dseg *)pdst; 2830 } 2831 pdst = (uint8_t *)txq->wqes; 2832 part = inlen - tlen; 2833 } while (true); 2834 } 2835 2836 /** 2837 * Build the Data Segment of pointer type. 2838 * 2839 * @param txq 2840 * Pointer to TX queue structure. 2841 * @param loc 2842 * Pointer to burst routine local context. 2843 * @param dseg 2844 * Pointer to WQE to fill with built Data Segment. 2845 * @param buf 2846 * Data buffer to point. 2847 * @param len 2848 * Data buffer length. 2849 * @param olx 2850 * Configured Tx offloads mask. It is fully defined at 2851 * compile time and may be used for optimization. 2852 */ 2853 static __rte_always_inline void 2854 mlx5_tx_dseg_ptr(struct mlx5_txq_data *restrict txq, 2855 struct mlx5_txq_local *restrict loc, 2856 struct mlx5_wqe_dseg *restrict dseg, 2857 uint8_t *buf, 2858 unsigned int len, 2859 unsigned int olx __rte_unused) 2860 2861 { 2862 MLX5_ASSERT(len); 2863 dseg->bcount = rte_cpu_to_be_32(len); 2864 dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf); 2865 dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf); 2866 } 2867 2868 /** 2869 * Build the Data Segment of pointer type or inline 2870 * if data length is less than buffer in minimal 2871 * Data Segment size. 2872 * 2873 * @param txq 2874 * Pointer to TX queue structure. 2875 * @param loc 2876 * Pointer to burst routine local context. 2877 * @param dseg 2878 * Pointer to WQE to fill with built Data Segment. 2879 * @param buf 2880 * Data buffer to point. 2881 * @param len 2882 * Data buffer length. 2883 * @param olx 2884 * Configured Tx offloads mask. It is fully defined at 2885 * compile time and may be used for optimization. 2886 */ 2887 static __rte_always_inline void 2888 mlx5_tx_dseg_iptr(struct mlx5_txq_data *restrict txq, 2889 struct mlx5_txq_local *restrict loc, 2890 struct mlx5_wqe_dseg *restrict dseg, 2891 uint8_t *buf, 2892 unsigned int len, 2893 unsigned int olx __rte_unused) 2894 2895 { 2896 uintptr_t dst, src; 2897 2898 MLX5_ASSERT(len); 2899 if (len > MLX5_DSEG_MIN_INLINE_SIZE) { 2900 dseg->bcount = rte_cpu_to_be_32(len); 2901 dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf); 2902 dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf); 2903 2904 return; 2905 } 2906 dseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE); 2907 /* Unrolled implementation of generic rte_memcpy. */ 2908 dst = (uintptr_t)&dseg->inline_data[0]; 2909 src = (uintptr_t)buf; 2910 if (len & 0x08) { 2911 #ifdef RTE_ARCH_STRICT_ALIGN 2912 MLX5_ASSERT(dst == RTE_PTR_ALIGN(dst, sizeof(uint32_t))); 2913 *(uint32_t *)dst = *(unaligned_uint32_t *)src; 2914 dst += sizeof(uint32_t); 2915 src += sizeof(uint32_t); 2916 *(uint32_t *)dst = *(unaligned_uint32_t *)src; 2917 dst += sizeof(uint32_t); 2918 src += sizeof(uint32_t); 2919 #else 2920 *(uint64_t *)dst = *(unaligned_uint64_t *)src; 2921 dst += sizeof(uint64_t); 2922 src += sizeof(uint64_t); 2923 #endif 2924 } 2925 if (len & 0x04) { 2926 *(uint32_t *)dst = *(unaligned_uint32_t *)src; 2927 dst += sizeof(uint32_t); 2928 src += sizeof(uint32_t); 2929 } 2930 if (len & 0x02) { 2931 *(uint16_t *)dst = *(unaligned_uint16_t *)src; 2932 dst += sizeof(uint16_t); 2933 src += sizeof(uint16_t); 2934 } 2935 if (len & 0x01) 2936 *(uint8_t *)dst = *(uint8_t *)src; 2937 } 2938 2939 /** 2940 * Build the Data Segment of inlined data from single 2941 * segment packet, no VLAN insertion. 2942 * 2943 * @param txq 2944 * Pointer to TX queue structure. 2945 * @param loc 2946 * Pointer to burst routine local context. 2947 * @param dseg 2948 * Pointer to WQE to fill with built Data Segment. 2949 * @param buf 2950 * Data buffer to point. 2951 * @param len 2952 * Data buffer length. 2953 * @param olx 2954 * Configured Tx offloads mask. It is fully defined at 2955 * compile time and may be used for optimization. 2956 * 2957 * @return 2958 * Pointer to the next Data Segment after inlined data. 2959 * Ring buffer wraparound check is needed. We do not 2960 * do it here because it may not be needed for the 2961 * last packet in the eMPW session. 2962 */ 2963 static __rte_always_inline struct mlx5_wqe_dseg * 2964 mlx5_tx_dseg_empw(struct mlx5_txq_data *restrict txq, 2965 struct mlx5_txq_local *restrict loc __rte_unused, 2966 struct mlx5_wqe_dseg *restrict dseg, 2967 uint8_t *buf, 2968 unsigned int len, 2969 unsigned int olx __rte_unused) 2970 { 2971 unsigned int part; 2972 uint8_t *pdst; 2973 2974 if (!MLX5_TXOFF_CONFIG(MPW)) { 2975 /* Store the descriptor byte counter for eMPW sessions. */ 2976 dseg->bcount = rte_cpu_to_be_32(len | MLX5_ETH_WQE_DATA_INLINE); 2977 pdst = &dseg->inline_data[0]; 2978 } else { 2979 /* The entire legacy MPW session counter is stored on close. */ 2980 pdst = (uint8_t *)dseg; 2981 } 2982 /* 2983 * The WQEBB space availability is checked by caller. 2984 * Here we should be aware of WQE ring buffer wraparound only. 2985 */ 2986 part = (uint8_t *)txq->wqes_end - pdst; 2987 part = RTE_MIN(part, len); 2988 do { 2989 rte_memcpy(pdst, buf, part); 2990 len -= part; 2991 if (likely(!len)) { 2992 pdst += part; 2993 if (!MLX5_TXOFF_CONFIG(MPW)) 2994 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 2995 /* Note: no final wraparound check here. */ 2996 return (struct mlx5_wqe_dseg *)pdst; 2997 } 2998 pdst = (uint8_t *)txq->wqes; 2999 buf += part; 3000 part = len; 3001 } while (true); 3002 } 3003 3004 /** 3005 * Build the Data Segment of inlined data from single 3006 * segment packet with VLAN insertion. 3007 * 3008 * @param txq 3009 * Pointer to TX queue structure. 3010 * @param loc 3011 * Pointer to burst routine local context. 3012 * @param dseg 3013 * Pointer to the dseg fill with built Data Segment. 3014 * @param buf 3015 * Data buffer to point. 3016 * @param len 3017 * Data buffer length. 3018 * @param olx 3019 * Configured Tx offloads mask. It is fully defined at 3020 * compile time and may be used for optimization. 3021 * 3022 * @return 3023 * Pointer to the next Data Segment after inlined data. 3024 * Ring buffer wraparound check is needed. 3025 */ 3026 static __rte_always_inline struct mlx5_wqe_dseg * 3027 mlx5_tx_dseg_vlan(struct mlx5_txq_data *restrict txq, 3028 struct mlx5_txq_local *restrict loc __rte_unused, 3029 struct mlx5_wqe_dseg *restrict dseg, 3030 uint8_t *buf, 3031 unsigned int len, 3032 unsigned int olx __rte_unused) 3033 3034 { 3035 unsigned int part; 3036 uint8_t *pdst; 3037 3038 MLX5_ASSERT(len > MLX5_ESEG_MIN_INLINE_SIZE); 3039 static_assert(MLX5_DSEG_MIN_INLINE_SIZE == 3040 (2 * RTE_ETHER_ADDR_LEN), 3041 "invalid Data Segment data size"); 3042 if (!MLX5_TXOFF_CONFIG(MPW)) { 3043 /* Store the descriptor byte counter for eMPW sessions. */ 3044 dseg->bcount = rte_cpu_to_be_32 3045 ((len + sizeof(struct rte_vlan_hdr)) | 3046 MLX5_ETH_WQE_DATA_INLINE); 3047 pdst = &dseg->inline_data[0]; 3048 } else { 3049 /* The entire legacy MPW session counter is stored on close. */ 3050 pdst = (uint8_t *)dseg; 3051 } 3052 memcpy(pdst, buf, MLX5_DSEG_MIN_INLINE_SIZE); 3053 buf += MLX5_DSEG_MIN_INLINE_SIZE; 3054 pdst += MLX5_DSEG_MIN_INLINE_SIZE; 3055 len -= MLX5_DSEG_MIN_INLINE_SIZE; 3056 /* Insert VLAN ethertype + VLAN tag. Pointer is aligned. */ 3057 MLX5_ASSERT(pdst == RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE)); 3058 if (unlikely(pdst >= (uint8_t *)txq->wqes_end)) 3059 pdst = (uint8_t *)txq->wqes; 3060 *(uint32_t *)pdst = rte_cpu_to_be_32((RTE_ETHER_TYPE_VLAN << 16) | 3061 loc->mbuf->vlan_tci); 3062 pdst += sizeof(struct rte_vlan_hdr); 3063 /* 3064 * The WQEBB space availability is checked by caller. 3065 * Here we should be aware of WQE ring buffer wraparound only. 3066 */ 3067 part = (uint8_t *)txq->wqes_end - pdst; 3068 part = RTE_MIN(part, len); 3069 do { 3070 rte_memcpy(pdst, buf, part); 3071 len -= part; 3072 if (likely(!len)) { 3073 pdst += part; 3074 if (!MLX5_TXOFF_CONFIG(MPW)) 3075 pdst = RTE_PTR_ALIGN(pdst, MLX5_WSEG_SIZE); 3076 /* Note: no final wraparound check here. */ 3077 return (struct mlx5_wqe_dseg *)pdst; 3078 } 3079 pdst = (uint8_t *)txq->wqes; 3080 buf += part; 3081 part = len; 3082 } while (true); 3083 } 3084 3085 /** 3086 * Build the Ethernet Segment with optionally inlined data with 3087 * VLAN insertion and following Data Segments (if any) from 3088 * multi-segment packet. Used by ordinary send and TSO. 3089 * 3090 * @param txq 3091 * Pointer to TX queue structure. 3092 * @param loc 3093 * Pointer to burst routine local context. 3094 * @param wqe 3095 * Pointer to WQE to fill with built Ethernet/Data Segments. 3096 * @param vlan 3097 * Length of VLAN header to insert, 0 means no VLAN insertion. 3098 * @param inlen 3099 * Data length to inline. For TSO this parameter specifies 3100 * exact value, for ordinary send routine can be aligned by 3101 * caller to provide better WQE space saving and data buffer 3102 * start address alignment. This length includes VLAN header 3103 * being inserted. 3104 * @param tso 3105 * Zero means ordinary send, inlined data can be extended, 3106 * otherwise this is TSO, inlined data length is fixed. 3107 * @param olx 3108 * Configured Tx offloads mask. It is fully defined at 3109 * compile time and may be used for optimization. 3110 * 3111 * @return 3112 * Actual size of built WQE in segments. 3113 */ 3114 static __rte_always_inline unsigned int 3115 mlx5_tx_mseg_build(struct mlx5_txq_data *restrict txq, 3116 struct mlx5_txq_local *restrict loc, 3117 struct mlx5_wqe *restrict wqe, 3118 unsigned int vlan, 3119 unsigned int inlen, 3120 unsigned int tso, 3121 unsigned int olx __rte_unused) 3122 { 3123 struct mlx5_wqe_dseg *restrict dseg; 3124 unsigned int ds; 3125 3126 MLX5_ASSERT((rte_pktmbuf_pkt_len(loc->mbuf) + vlan) >= inlen); 3127 loc->mbuf_nseg = NB_SEGS(loc->mbuf); 3128 loc->mbuf_off = 0; 3129 3130 dseg = mlx5_tx_eseg_mdat(txq, loc, wqe, vlan, inlen, tso, olx); 3131 if (!loc->mbuf_nseg) 3132 goto dseg_done; 3133 /* 3134 * There are still some mbuf remaining, not inlined. 3135 * The first mbuf may be partially inlined and we 3136 * must process the possible non-zero data offset. 3137 */ 3138 if (loc->mbuf_off) { 3139 unsigned int dlen; 3140 uint8_t *dptr; 3141 3142 /* 3143 * Exhausted packets must be dropped before. 3144 * Non-zero offset means there are some data 3145 * remained in the packet. 3146 */ 3147 MLX5_ASSERT(loc->mbuf_off < rte_pktmbuf_data_len(loc->mbuf)); 3148 MLX5_ASSERT(rte_pktmbuf_data_len(loc->mbuf)); 3149 dptr = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *, 3150 loc->mbuf_off); 3151 dlen = rte_pktmbuf_data_len(loc->mbuf) - loc->mbuf_off; 3152 /* 3153 * Build the pointer/minimal data Data Segment. 3154 * Do ring buffer wrapping check in advance. 3155 */ 3156 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 3157 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 3158 mlx5_tx_dseg_iptr(txq, loc, dseg, dptr, dlen, olx); 3159 /* Store the mbuf to be freed on completion. */ 3160 MLX5_ASSERT(loc->elts_free); 3161 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3162 --loc->elts_free; 3163 ++dseg; 3164 if (--loc->mbuf_nseg == 0) 3165 goto dseg_done; 3166 loc->mbuf = loc->mbuf->next; 3167 loc->mbuf_off = 0; 3168 } 3169 do { 3170 if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) { 3171 struct rte_mbuf *mbuf; 3172 3173 /* Zero length segment found, just skip. */ 3174 mbuf = loc->mbuf; 3175 loc->mbuf = loc->mbuf->next; 3176 rte_pktmbuf_free_seg(mbuf); 3177 if (--loc->mbuf_nseg == 0) 3178 break; 3179 } else { 3180 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 3181 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 3182 mlx5_tx_dseg_iptr 3183 (txq, loc, dseg, 3184 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 3185 rte_pktmbuf_data_len(loc->mbuf), olx); 3186 MLX5_ASSERT(loc->elts_free); 3187 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3188 --loc->elts_free; 3189 ++dseg; 3190 if (--loc->mbuf_nseg == 0) 3191 break; 3192 loc->mbuf = loc->mbuf->next; 3193 } 3194 } while (true); 3195 3196 dseg_done: 3197 /* Calculate actual segments used from the dseg pointer. */ 3198 if ((uintptr_t)wqe < (uintptr_t)dseg) 3199 ds = ((uintptr_t)dseg - (uintptr_t)wqe) / MLX5_WSEG_SIZE; 3200 else 3201 ds = (((uintptr_t)dseg - (uintptr_t)wqe) + 3202 txq->wqe_s * MLX5_WQE_SIZE) / MLX5_WSEG_SIZE; 3203 return ds; 3204 } 3205 3206 /** 3207 * Tx one packet function for multi-segment TSO. Supports all 3208 * types of Tx offloads, uses MLX5_OPCODE_TSO to build WQEs, 3209 * sends one packet per WQE. 3210 * 3211 * This routine is responsible for storing processed mbuf 3212 * into elts ring buffer and update elts_head. 3213 * 3214 * @param txq 3215 * Pointer to TX queue structure. 3216 * @param loc 3217 * Pointer to burst routine local context. 3218 * @param olx 3219 * Configured Tx offloads mask. It is fully defined at 3220 * compile time and may be used for optimization. 3221 * 3222 * @return 3223 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3224 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3225 * Local context variables partially updated. 3226 */ 3227 static __rte_always_inline enum mlx5_txcmp_code 3228 mlx5_tx_packet_multi_tso(struct mlx5_txq_data *restrict txq, 3229 struct mlx5_txq_local *restrict loc, 3230 unsigned int olx) 3231 { 3232 struct mlx5_wqe *restrict wqe; 3233 unsigned int ds, dlen, inlen, ntcp, vlan = 0; 3234 3235 /* 3236 * Calculate data length to be inlined to estimate 3237 * the required space in WQE ring buffer. 3238 */ 3239 dlen = rte_pktmbuf_pkt_len(loc->mbuf); 3240 if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 3241 vlan = sizeof(struct rte_vlan_hdr); 3242 inlen = loc->mbuf->l2_len + vlan + 3243 loc->mbuf->l3_len + loc->mbuf->l4_len; 3244 if (unlikely((!inlen || !loc->mbuf->tso_segsz))) 3245 return MLX5_TXCMP_CODE_ERROR; 3246 if (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK) 3247 inlen += loc->mbuf->outer_l2_len + loc->mbuf->outer_l3_len; 3248 /* Packet must contain all TSO headers. */ 3249 if (unlikely(inlen > MLX5_MAX_TSO_HEADER || 3250 inlen <= MLX5_ESEG_MIN_INLINE_SIZE || 3251 inlen > (dlen + vlan))) 3252 return MLX5_TXCMP_CODE_ERROR; 3253 MLX5_ASSERT(inlen >= txq->inlen_mode); 3254 /* 3255 * Check whether there are enough free WQEBBs: 3256 * - Control Segment 3257 * - Ethernet Segment 3258 * - First Segment of inlined Ethernet data 3259 * - ... data continued ... 3260 * - Data Segments of pointer/min inline type 3261 */ 3262 ds = NB_SEGS(loc->mbuf) + 2 + (inlen - 3263 MLX5_ESEG_MIN_INLINE_SIZE + 3264 MLX5_WSEG_SIZE + 3265 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 3266 if (unlikely(loc->wqe_free < ((ds + 3) / 4))) 3267 return MLX5_TXCMP_CODE_EXIT; 3268 /* Check for maximal WQE size. */ 3269 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4))) 3270 return MLX5_TXCMP_CODE_ERROR; 3271 #ifdef MLX5_PMD_SOFT_COUNTERS 3272 /* Update sent data bytes/packets counters. */ 3273 ntcp = (dlen - (inlen - vlan) + loc->mbuf->tso_segsz - 1) / 3274 loc->mbuf->tso_segsz; 3275 /* 3276 * One will be added for mbuf itself 3277 * at the end of the mlx5_tx_burst from 3278 * loc->pkts_sent field. 3279 */ 3280 --ntcp; 3281 txq->stats.opackets += ntcp; 3282 txq->stats.obytes += dlen + vlan + ntcp * inlen; 3283 #endif 3284 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3285 loc->wqe_last = wqe; 3286 mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_TSO, olx); 3287 ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 1, olx); 3288 wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 3289 txq->wqe_ci += (ds + 3) / 4; 3290 loc->wqe_free -= (ds + 3) / 4; 3291 return MLX5_TXCMP_CODE_MULTI; 3292 } 3293 3294 /** 3295 * Tx one packet function for multi-segment SEND. Supports all 3296 * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs, 3297 * sends one packet per WQE, without any data inlining in 3298 * Ethernet Segment. 3299 * 3300 * This routine is responsible for storing processed mbuf 3301 * into elts ring buffer and update elts_head. 3302 * 3303 * @param txq 3304 * Pointer to TX queue structure. 3305 * @param loc 3306 * Pointer to burst routine local context. 3307 * @param olx 3308 * Configured Tx offloads mask. It is fully defined at 3309 * compile time and may be used for optimization. 3310 * 3311 * @return 3312 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3313 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3314 * Local context variables partially updated. 3315 */ 3316 static __rte_always_inline enum mlx5_txcmp_code 3317 mlx5_tx_packet_multi_send(struct mlx5_txq_data *restrict txq, 3318 struct mlx5_txq_local *restrict loc, 3319 unsigned int olx) 3320 { 3321 struct mlx5_wqe_dseg *restrict dseg; 3322 struct mlx5_wqe *restrict wqe; 3323 unsigned int ds, nseg; 3324 3325 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1); 3326 /* 3327 * No inline at all, it means the CPU cycles saving 3328 * is prioritized at configuration, we should not 3329 * copy any packet data to WQE. 3330 */ 3331 nseg = NB_SEGS(loc->mbuf); 3332 ds = 2 + nseg; 3333 if (unlikely(loc->wqe_free < ((ds + 3) / 4))) 3334 return MLX5_TXCMP_CODE_EXIT; 3335 /* Check for maximal WQE size. */ 3336 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4))) 3337 return MLX5_TXCMP_CODE_ERROR; 3338 /* 3339 * Some Tx offloads may cause an error if 3340 * packet is not long enough, check against 3341 * assumed minimal length. 3342 */ 3343 if (rte_pktmbuf_pkt_len(loc->mbuf) <= MLX5_ESEG_MIN_INLINE_SIZE) 3344 return MLX5_TXCMP_CODE_ERROR; 3345 #ifdef MLX5_PMD_SOFT_COUNTERS 3346 /* Update sent data bytes counter. */ 3347 txq->stats.obytes += rte_pktmbuf_pkt_len(loc->mbuf); 3348 if (MLX5_TXOFF_CONFIG(VLAN) && 3349 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 3350 txq->stats.obytes += sizeof(struct rte_vlan_hdr); 3351 #endif 3352 /* 3353 * SEND WQE, one WQEBB: 3354 * - Control Segment, SEND opcode 3355 * - Ethernet Segment, optional VLAN, no inline 3356 * - Data Segments, pointer only type 3357 */ 3358 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3359 loc->wqe_last = wqe; 3360 mlx5_tx_cseg_init(txq, loc, wqe, ds, MLX5_OPCODE_SEND, olx); 3361 mlx5_tx_eseg_none(txq, loc, wqe, olx); 3362 dseg = &wqe->dseg[0]; 3363 do { 3364 if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) { 3365 struct rte_mbuf *mbuf; 3366 3367 /* 3368 * Zero length segment found, have to 3369 * correct total size of WQE in segments. 3370 * It is supposed to be rare occasion, so 3371 * in normal case (no zero length segments) 3372 * we avoid extra writing to the Control 3373 * Segment. 3374 */ 3375 --ds; 3376 wqe->cseg.sq_ds -= RTE_BE32(1); 3377 mbuf = loc->mbuf; 3378 loc->mbuf = mbuf->next; 3379 rte_pktmbuf_free_seg(mbuf); 3380 if (--nseg == 0) 3381 break; 3382 } else { 3383 mlx5_tx_dseg_ptr 3384 (txq, loc, dseg, 3385 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 3386 rte_pktmbuf_data_len(loc->mbuf), olx); 3387 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3388 --loc->elts_free; 3389 if (--nseg == 0) 3390 break; 3391 ++dseg; 3392 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 3393 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 3394 loc->mbuf = loc->mbuf->next; 3395 } 3396 } while (true); 3397 txq->wqe_ci += (ds + 3) / 4; 3398 loc->wqe_free -= (ds + 3) / 4; 3399 return MLX5_TXCMP_CODE_MULTI; 3400 } 3401 3402 /** 3403 * Tx one packet function for multi-segment SEND. Supports all 3404 * types of Tx offloads, uses MLX5_OPCODE_SEND to build WQEs, 3405 * sends one packet per WQE, with data inlining in 3406 * Ethernet Segment and minimal Data Segments. 3407 * 3408 * This routine is responsible for storing processed mbuf 3409 * into elts ring buffer and update elts_head. 3410 * 3411 * @param txq 3412 * Pointer to TX queue structure. 3413 * @param loc 3414 * Pointer to burst routine local context. 3415 * @param olx 3416 * Configured Tx offloads mask. It is fully defined at 3417 * compile time and may be used for optimization. 3418 * 3419 * @return 3420 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3421 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3422 * Local context variables partially updated. 3423 */ 3424 static __rte_always_inline enum mlx5_txcmp_code 3425 mlx5_tx_packet_multi_inline(struct mlx5_txq_data *restrict txq, 3426 struct mlx5_txq_local *restrict loc, 3427 unsigned int olx) 3428 { 3429 struct mlx5_wqe *restrict wqe; 3430 unsigned int ds, inlen, dlen, vlan = 0; 3431 3432 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 3433 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1); 3434 /* 3435 * First calculate data length to be inlined 3436 * to estimate the required space for WQE. 3437 */ 3438 dlen = rte_pktmbuf_pkt_len(loc->mbuf); 3439 if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 3440 vlan = sizeof(struct rte_vlan_hdr); 3441 inlen = dlen + vlan; 3442 /* Check against minimal length. */ 3443 if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE) 3444 return MLX5_TXCMP_CODE_ERROR; 3445 MLX5_ASSERT(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE); 3446 if (inlen > txq->inlen_send || 3447 loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) { 3448 struct rte_mbuf *mbuf; 3449 unsigned int nxlen; 3450 uintptr_t start; 3451 3452 /* 3453 * Packet length exceeds the allowed inline 3454 * data length, check whether the minimal 3455 * inlining is required. 3456 */ 3457 if (txq->inlen_mode) { 3458 MLX5_ASSERT(txq->inlen_mode >= 3459 MLX5_ESEG_MIN_INLINE_SIZE); 3460 MLX5_ASSERT(txq->inlen_mode <= txq->inlen_send); 3461 inlen = txq->inlen_mode; 3462 } else { 3463 if (loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE || 3464 !vlan || txq->vlan_en) { 3465 /* 3466 * VLAN insertion will be done inside by HW. 3467 * It is not utmost effective - VLAN flag is 3468 * checked twice, but we should proceed the 3469 * inlining length correctly and take into 3470 * account the VLAN header being inserted. 3471 */ 3472 return mlx5_tx_packet_multi_send 3473 (txq, loc, olx); 3474 } 3475 inlen = MLX5_ESEG_MIN_INLINE_SIZE; 3476 } 3477 /* 3478 * Now we know the minimal amount of data is requested 3479 * to inline. Check whether we should inline the buffers 3480 * from the chain beginning to eliminate some mbufs. 3481 */ 3482 mbuf = loc->mbuf; 3483 nxlen = rte_pktmbuf_data_len(mbuf); 3484 if (unlikely(nxlen <= txq->inlen_send)) { 3485 /* We can inline first mbuf at least. */ 3486 if (nxlen < inlen) { 3487 unsigned int smlen; 3488 3489 /* Scan mbufs till inlen filled. */ 3490 do { 3491 smlen = nxlen; 3492 mbuf = NEXT(mbuf); 3493 MLX5_ASSERT(mbuf); 3494 nxlen = rte_pktmbuf_data_len(mbuf); 3495 nxlen += smlen; 3496 } while (unlikely(nxlen < inlen)); 3497 if (unlikely(nxlen > txq->inlen_send)) { 3498 /* We cannot inline entire mbuf. */ 3499 smlen = inlen - smlen; 3500 start = rte_pktmbuf_mtod_offset 3501 (mbuf, uintptr_t, smlen); 3502 goto do_align; 3503 } 3504 } 3505 do { 3506 inlen = nxlen; 3507 mbuf = NEXT(mbuf); 3508 /* There should be not end of packet. */ 3509 MLX5_ASSERT(mbuf); 3510 nxlen = inlen + rte_pktmbuf_data_len(mbuf); 3511 } while (unlikely(nxlen < txq->inlen_send)); 3512 } 3513 start = rte_pktmbuf_mtod(mbuf, uintptr_t); 3514 /* 3515 * Check whether we can do inline to align start 3516 * address of data buffer to cacheline. 3517 */ 3518 do_align: 3519 start = (~start + 1) & (RTE_CACHE_LINE_SIZE - 1); 3520 if (unlikely(start)) { 3521 start += inlen; 3522 if (start <= txq->inlen_send) 3523 inlen = start; 3524 } 3525 } 3526 /* 3527 * Check whether there are enough free WQEBBs: 3528 * - Control Segment 3529 * - Ethernet Segment 3530 * - First Segment of inlined Ethernet data 3531 * - ... data continued ... 3532 * - Data Segments of pointer/min inline type 3533 * 3534 * Estimate the number of Data Segments conservatively, 3535 * supposing no any mbufs is being freed during inlining. 3536 */ 3537 MLX5_ASSERT(inlen <= txq->inlen_send); 3538 ds = NB_SEGS(loc->mbuf) + 2 + (inlen - 3539 MLX5_ESEG_MIN_INLINE_SIZE + 3540 MLX5_WSEG_SIZE + 3541 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 3542 if (unlikely(loc->wqe_free < ((ds + 3) / 4))) 3543 return MLX5_TXCMP_CODE_EXIT; 3544 /* Check for maximal WQE size. */ 3545 if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4))) 3546 return MLX5_TXCMP_CODE_ERROR; 3547 #ifdef MLX5_PMD_SOFT_COUNTERS 3548 /* Update sent data bytes/packets counters. */ 3549 txq->stats.obytes += dlen + vlan; 3550 #endif 3551 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3552 loc->wqe_last = wqe; 3553 mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_SEND, olx); 3554 ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 0, olx); 3555 wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 3556 txq->wqe_ci += (ds + 3) / 4; 3557 loc->wqe_free -= (ds + 3) / 4; 3558 return MLX5_TXCMP_CODE_MULTI; 3559 } 3560 3561 /** 3562 * Tx burst function for multi-segment packets. Supports all 3563 * types of Tx offloads, uses MLX5_OPCODE_SEND/TSO to build WQEs, 3564 * sends one packet per WQE. Function stops sending if it 3565 * encounters the single-segment packet. 3566 * 3567 * This routine is responsible for storing processed mbuf 3568 * into elts ring buffer and update elts_head. 3569 * 3570 * @param txq 3571 * Pointer to TX queue structure. 3572 * @param[in] pkts 3573 * Packets to transmit. 3574 * @param pkts_n 3575 * Number of packets in array. 3576 * @param loc 3577 * Pointer to burst routine local context. 3578 * @param olx 3579 * Configured Tx offloads mask. It is fully defined at 3580 * compile time and may be used for optimization. 3581 * 3582 * @return 3583 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3584 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3585 * MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered. 3586 * MLX5_TXCMP_CODE_TSO - TSO single-segment packet encountered. 3587 * Local context variables updated. 3588 */ 3589 static __rte_always_inline enum mlx5_txcmp_code 3590 mlx5_tx_burst_mseg(struct mlx5_txq_data *restrict txq, 3591 struct rte_mbuf **restrict pkts, 3592 unsigned int pkts_n, 3593 struct mlx5_txq_local *restrict loc, 3594 unsigned int olx) 3595 { 3596 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 3597 MLX5_ASSERT(pkts_n > loc->pkts_sent); 3598 pkts += loc->pkts_sent + 1; 3599 pkts_n -= loc->pkts_sent; 3600 for (;;) { 3601 enum mlx5_txcmp_code ret; 3602 3603 MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1); 3604 /* 3605 * Estimate the number of free elts quickly but 3606 * conservatively. Some segment may be fully inlined 3607 * and freed, ignore this here - precise estimation 3608 * is costly. 3609 */ 3610 if (loc->elts_free < NB_SEGS(loc->mbuf)) 3611 return MLX5_TXCMP_CODE_EXIT; 3612 if (MLX5_TXOFF_CONFIG(TSO) && 3613 unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) { 3614 /* Proceed with multi-segment TSO. */ 3615 ret = mlx5_tx_packet_multi_tso(txq, loc, olx); 3616 } else if (MLX5_TXOFF_CONFIG(INLINE)) { 3617 /* Proceed with multi-segment SEND with inlining. */ 3618 ret = mlx5_tx_packet_multi_inline(txq, loc, olx); 3619 } else { 3620 /* Proceed with multi-segment SEND w/o inlining. */ 3621 ret = mlx5_tx_packet_multi_send(txq, loc, olx); 3622 } 3623 if (ret == MLX5_TXCMP_CODE_EXIT) 3624 return MLX5_TXCMP_CODE_EXIT; 3625 if (ret == MLX5_TXCMP_CODE_ERROR) 3626 return MLX5_TXCMP_CODE_ERROR; 3627 /* WQE is built, go to the next packet. */ 3628 ++loc->pkts_sent; 3629 --pkts_n; 3630 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 3631 return MLX5_TXCMP_CODE_EXIT; 3632 loc->mbuf = *pkts++; 3633 if (pkts_n > 1) 3634 rte_prefetch0(*pkts); 3635 if (likely(NB_SEGS(loc->mbuf) > 1)) 3636 continue; 3637 /* Here ends the series of multi-segment packets. */ 3638 if (MLX5_TXOFF_CONFIG(TSO) && 3639 unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) 3640 return MLX5_TXCMP_CODE_TSO; 3641 return MLX5_TXCMP_CODE_SINGLE; 3642 } 3643 MLX5_ASSERT(false); 3644 } 3645 3646 /** 3647 * Tx burst function for single-segment packets with TSO. 3648 * Supports all types of Tx offloads, except multi-packets. 3649 * Uses MLX5_OPCODE_TSO to build WQEs, sends one packet per WQE. 3650 * Function stops sending if it encounters the multi-segment 3651 * packet or packet without TSO requested. 3652 * 3653 * The routine is responsible for storing processed mbuf 3654 * into elts ring buffer and update elts_head if inline 3655 * offloads is requested due to possible early freeing 3656 * of the inlined mbufs (can not store pkts array in elts 3657 * as a batch). 3658 * 3659 * @param txq 3660 * Pointer to TX queue structure. 3661 * @param[in] pkts 3662 * Packets to transmit. 3663 * @param pkts_n 3664 * Number of packets in array. 3665 * @param loc 3666 * Pointer to burst routine local context. 3667 * @param olx 3668 * Configured Tx offloads mask. It is fully defined at 3669 * compile time and may be used for optimization. 3670 * 3671 * @return 3672 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 3673 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 3674 * MLX5_TXCMP_CODE_SINGLE - single-segment packet encountered. 3675 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered. 3676 * Local context variables updated. 3677 */ 3678 static __rte_always_inline enum mlx5_txcmp_code 3679 mlx5_tx_burst_tso(struct mlx5_txq_data *restrict txq, 3680 struct rte_mbuf **restrict pkts, 3681 unsigned int pkts_n, 3682 struct mlx5_txq_local *restrict loc, 3683 unsigned int olx) 3684 { 3685 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 3686 MLX5_ASSERT(pkts_n > loc->pkts_sent); 3687 pkts += loc->pkts_sent + 1; 3688 pkts_n -= loc->pkts_sent; 3689 for (;;) { 3690 struct mlx5_wqe_dseg *restrict dseg; 3691 struct mlx5_wqe *restrict wqe; 3692 unsigned int ds, dlen, hlen, ntcp, vlan = 0; 3693 uint8_t *dptr; 3694 3695 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 3696 dlen = rte_pktmbuf_data_len(loc->mbuf); 3697 if (MLX5_TXOFF_CONFIG(VLAN) && 3698 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 3699 vlan = sizeof(struct rte_vlan_hdr); 3700 } 3701 /* 3702 * First calculate the WQE size to check 3703 * whether we have enough space in ring buffer. 3704 */ 3705 hlen = loc->mbuf->l2_len + vlan + 3706 loc->mbuf->l3_len + loc->mbuf->l4_len; 3707 if (unlikely((!hlen || !loc->mbuf->tso_segsz))) 3708 return MLX5_TXCMP_CODE_ERROR; 3709 if (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK) 3710 hlen += loc->mbuf->outer_l2_len + 3711 loc->mbuf->outer_l3_len; 3712 /* Segment must contain all TSO headers. */ 3713 if (unlikely(hlen > MLX5_MAX_TSO_HEADER || 3714 hlen <= MLX5_ESEG_MIN_INLINE_SIZE || 3715 hlen > (dlen + vlan))) 3716 return MLX5_TXCMP_CODE_ERROR; 3717 /* 3718 * Check whether there are enough free WQEBBs: 3719 * - Control Segment 3720 * - Ethernet Segment 3721 * - First Segment of inlined Ethernet data 3722 * - ... data continued ... 3723 * - Finishing Data Segment of pointer type 3724 */ 3725 ds = 4 + (hlen - MLX5_ESEG_MIN_INLINE_SIZE + 3726 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 3727 if (loc->wqe_free < ((ds + 3) / 4)) 3728 return MLX5_TXCMP_CODE_EXIT; 3729 #ifdef MLX5_PMD_SOFT_COUNTERS 3730 /* Update sent data bytes/packets counters. */ 3731 ntcp = (dlen + vlan - hlen + 3732 loc->mbuf->tso_segsz - 1) / 3733 loc->mbuf->tso_segsz; 3734 /* 3735 * One will be added for mbuf itself at the end 3736 * of the mlx5_tx_burst from loc->pkts_sent field. 3737 */ 3738 --ntcp; 3739 txq->stats.opackets += ntcp; 3740 txq->stats.obytes += dlen + vlan + ntcp * hlen; 3741 #endif 3742 /* 3743 * Build the TSO WQE: 3744 * - Control Segment 3745 * - Ethernet Segment with hlen bytes inlined 3746 * - Data Segment of pointer type 3747 */ 3748 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 3749 loc->wqe_last = wqe; 3750 mlx5_tx_cseg_init(txq, loc, wqe, ds, 3751 MLX5_OPCODE_TSO, olx); 3752 dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, hlen, 1, olx); 3753 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + hlen - vlan; 3754 dlen -= hlen - vlan; 3755 mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx); 3756 /* 3757 * WQE is built, update the loop parameters 3758 * and go to the next packet. 3759 */ 3760 txq->wqe_ci += (ds + 3) / 4; 3761 loc->wqe_free -= (ds + 3) / 4; 3762 if (MLX5_TXOFF_CONFIG(INLINE)) 3763 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 3764 --loc->elts_free; 3765 ++loc->pkts_sent; 3766 --pkts_n; 3767 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 3768 return MLX5_TXCMP_CODE_EXIT; 3769 loc->mbuf = *pkts++; 3770 if (pkts_n > 1) 3771 rte_prefetch0(*pkts); 3772 if (MLX5_TXOFF_CONFIG(MULTI) && 3773 unlikely(NB_SEGS(loc->mbuf) > 1)) 3774 return MLX5_TXCMP_CODE_MULTI; 3775 if (likely(!(loc->mbuf->ol_flags & PKT_TX_TCP_SEG))) 3776 return MLX5_TXCMP_CODE_SINGLE; 3777 /* Continue with the next TSO packet. */ 3778 } 3779 MLX5_ASSERT(false); 3780 } 3781 3782 /** 3783 * Analyze the packet and select the best method to send. 3784 * 3785 * @param txq 3786 * Pointer to TX queue structure. 3787 * @param loc 3788 * Pointer to burst routine local context. 3789 * @param olx 3790 * Configured Tx offloads mask. It is fully defined at 3791 * compile time and may be used for optimization. 3792 * @param newp 3793 * The predefined flag whether do complete check for 3794 * multi-segment packets and TSO. 3795 * 3796 * @return 3797 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered. 3798 * MLX5_TXCMP_CODE_TSO - TSO required, use TSO/LSO. 3799 * MLX5_TXCMP_CODE_SINGLE - single-segment packet, use SEND. 3800 * MLX5_TXCMP_CODE_EMPW - single-segment packet, use MPW. 3801 */ 3802 static __rte_always_inline enum mlx5_txcmp_code 3803 mlx5_tx_able_to_empw(struct mlx5_txq_data *restrict txq, 3804 struct mlx5_txq_local *restrict loc, 3805 unsigned int olx, 3806 bool newp) 3807 { 3808 /* Check for multi-segment packet. */ 3809 if (newp && 3810 MLX5_TXOFF_CONFIG(MULTI) && 3811 unlikely(NB_SEGS(loc->mbuf) > 1)) 3812 return MLX5_TXCMP_CODE_MULTI; 3813 /* Check for TSO packet. */ 3814 if (newp && 3815 MLX5_TXOFF_CONFIG(TSO) && 3816 unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)) 3817 return MLX5_TXCMP_CODE_TSO; 3818 /* Check if eMPW is enabled at all. */ 3819 if (!MLX5_TXOFF_CONFIG(EMPW)) 3820 return MLX5_TXCMP_CODE_SINGLE; 3821 /* Check if eMPW can be engaged. */ 3822 if (MLX5_TXOFF_CONFIG(VLAN) && 3823 unlikely(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) && 3824 (!MLX5_TXOFF_CONFIG(INLINE) || 3825 unlikely((rte_pktmbuf_data_len(loc->mbuf) + 3826 sizeof(struct rte_vlan_hdr)) > txq->inlen_empw))) { 3827 /* 3828 * eMPW does not support VLAN insertion offload, 3829 * we have to inline the entire packet but 3830 * packet is too long for inlining. 3831 */ 3832 return MLX5_TXCMP_CODE_SINGLE; 3833 } 3834 return MLX5_TXCMP_CODE_EMPW; 3835 } 3836 3837 /** 3838 * Check the next packet attributes to match with the eMPW batch ones. 3839 * In addition, for legacy MPW the packet length is checked either. 3840 * 3841 * @param txq 3842 * Pointer to TX queue structure. 3843 * @param es 3844 * Pointer to Ethernet Segment of eMPW batch. 3845 * @param loc 3846 * Pointer to burst routine local context. 3847 * @param dlen 3848 * Length of previous packet in MPW descriptor. 3849 * @param olx 3850 * Configured Tx offloads mask. It is fully defined at 3851 * compile time and may be used for optimization. 3852 * 3853 * @return 3854 * true - packet match with eMPW batch attributes. 3855 * false - no match, eMPW should be restarted. 3856 */ 3857 static __rte_always_inline bool 3858 mlx5_tx_match_empw(struct mlx5_txq_data *restrict txq __rte_unused, 3859 struct mlx5_wqe_eseg *restrict es, 3860 struct mlx5_txq_local *restrict loc, 3861 uint32_t dlen, 3862 unsigned int olx) 3863 { 3864 uint8_t swp_flags = 0; 3865 3866 /* Compare the checksum flags, if any. */ 3867 if (MLX5_TXOFF_CONFIG(CSUM) && 3868 txq_ol_cksum_to_cs(loc->mbuf) != es->cs_flags) 3869 return false; 3870 /* Compare the Software Parser offsets and flags. */ 3871 if (MLX5_TXOFF_CONFIG(SWP) && 3872 (es->swp_offs != txq_mbuf_to_swp(loc, &swp_flags, olx) || 3873 es->swp_flags != swp_flags)) 3874 return false; 3875 /* Fill metadata field if needed. */ 3876 if (MLX5_TXOFF_CONFIG(METADATA) && 3877 es->metadata != (loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ? 3878 *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0)) 3879 return false; 3880 /* Legacy MPW can send packets with the same lengt only. */ 3881 if (MLX5_TXOFF_CONFIG(MPW) && 3882 dlen != rte_pktmbuf_data_len(loc->mbuf)) 3883 return false; 3884 /* There must be no VLAN packets in eMPW loop. */ 3885 if (MLX5_TXOFF_CONFIG(VLAN)) 3886 MLX5_ASSERT(!(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)); 3887 return true; 3888 } 3889 3890 /* 3891 * Update send loop variables and WQE for eMPW loop 3892 * without data inlining. Number of Data Segments is 3893 * equal to the number of sent packets. 3894 * 3895 * @param txq 3896 * Pointer to TX queue structure. 3897 * @param loc 3898 * Pointer to burst routine local context. 3899 * @param ds 3900 * Number of packets/Data Segments/Packets. 3901 * @param slen 3902 * Accumulated statistics, bytes sent 3903 * @param olx 3904 * Configured Tx offloads mask. It is fully defined at 3905 * compile time and may be used for optimization. 3906 * 3907 * @return 3908 * true - packet match with eMPW batch attributes. 3909 * false - no match, eMPW should be restarted. 3910 */ 3911 static __rte_always_inline void 3912 mlx5_tx_sdone_empw(struct mlx5_txq_data *restrict txq, 3913 struct mlx5_txq_local *restrict loc, 3914 unsigned int ds, 3915 unsigned int slen, 3916 unsigned int olx __rte_unused) 3917 { 3918 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE)); 3919 #ifdef MLX5_PMD_SOFT_COUNTERS 3920 /* Update sent data bytes counter. */ 3921 txq->stats.obytes += slen; 3922 #else 3923 (void)slen; 3924 #endif 3925 loc->elts_free -= ds; 3926 loc->pkts_sent += ds; 3927 ds += 2; 3928 loc->wqe_last->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds); 3929 txq->wqe_ci += (ds + 3) / 4; 3930 loc->wqe_free -= (ds + 3) / 4; 3931 } 3932 3933 /* 3934 * Update send loop variables and WQE for eMPW loop 3935 * with data inlining. Gets the size of pushed descriptors 3936 * and data to the WQE. 3937 * 3938 * @param txq 3939 * Pointer to TX queue structure. 3940 * @param loc 3941 * Pointer to burst routine local context. 3942 * @param len 3943 * Total size of descriptor/data in bytes. 3944 * @param slen 3945 * Accumulated statistics, data bytes sent. 3946 * @param wqem 3947 * The base WQE for the eMPW/MPW descriptor. 3948 * @param olx 3949 * Configured Tx offloads mask. It is fully defined at 3950 * compile time and may be used for optimization. 3951 * 3952 * @return 3953 * true - packet match with eMPW batch attributes. 3954 * false - no match, eMPW should be restarted. 3955 */ 3956 static __rte_always_inline void 3957 mlx5_tx_idone_empw(struct mlx5_txq_data *restrict txq, 3958 struct mlx5_txq_local *restrict loc, 3959 unsigned int len, 3960 unsigned int slen, 3961 struct mlx5_wqe *restrict wqem, 3962 unsigned int olx __rte_unused) 3963 { 3964 struct mlx5_wqe_dseg *dseg = &wqem->dseg[0]; 3965 3966 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 3967 #ifdef MLX5_PMD_SOFT_COUNTERS 3968 /* Update sent data bytes counter. */ 3969 txq->stats.obytes += slen; 3970 #else 3971 (void)slen; 3972 #endif 3973 if (MLX5_TXOFF_CONFIG(MPW) && dseg->bcount == RTE_BE32(0)) { 3974 /* 3975 * If the legacy MPW session contains the inline packets 3976 * we should set the only inline data segment length 3977 * and align the total length to the segment size. 3978 */ 3979 MLX5_ASSERT(len > sizeof(dseg->bcount)); 3980 dseg->bcount = rte_cpu_to_be_32((len - sizeof(dseg->bcount)) | 3981 MLX5_ETH_WQE_DATA_INLINE); 3982 len = (len + MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE + 2; 3983 } else { 3984 /* 3985 * The session is not legacy MPW or contains the 3986 * data buffer pointer segments. 3987 */ 3988 MLX5_ASSERT((len % MLX5_WSEG_SIZE) == 0); 3989 len = len / MLX5_WSEG_SIZE + 2; 3990 } 3991 wqem->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | len); 3992 txq->wqe_ci += (len + 3) / 4; 3993 loc->wqe_free -= (len + 3) / 4; 3994 loc->wqe_last = wqem; 3995 } 3996 3997 /** 3998 * The set of Tx burst functions for single-segment packets 3999 * without TSO and with Multi-Packet Writing feature support. 4000 * Supports all types of Tx offloads, except multi-packets 4001 * and TSO. 4002 * 4003 * Uses MLX5_OPCODE_EMPW to build WQEs if possible and sends 4004 * as many packet per WQE as it can. If eMPW is not configured 4005 * or packet can not be sent with eMPW (VLAN insertion) the 4006 * ordinary SEND opcode is used and only one packet placed 4007 * in WQE. 4008 * 4009 * Functions stop sending if it encounters the multi-segment 4010 * packet or packet with TSO requested. 4011 * 4012 * The routines are responsible for storing processed mbuf 4013 * into elts ring buffer and update elts_head if inlining 4014 * offload is requested. Otherwise the copying mbufs to elts 4015 * can be postponed and completed at the end of burst routine. 4016 * 4017 * @param txq 4018 * Pointer to TX queue structure. 4019 * @param[in] pkts 4020 * Packets to transmit. 4021 * @param pkts_n 4022 * Number of packets in array. 4023 * @param loc 4024 * Pointer to burst routine local context. 4025 * @param olx 4026 * Configured Tx offloads mask. It is fully defined at 4027 * compile time and may be used for optimization. 4028 * 4029 * @return 4030 * MLX5_TXCMP_CODE_EXIT - sending is done or impossible. 4031 * MLX5_TXCMP_CODE_ERROR - some unrecoverable error occurred. 4032 * MLX5_TXCMP_CODE_MULTI - multi-segment packet encountered. 4033 * MLX5_TXCMP_CODE_TSO - TSO packet encountered. 4034 * MLX5_TXCMP_CODE_SINGLE - used inside functions set. 4035 * MLX5_TXCMP_CODE_EMPW - used inside functions set. 4036 * 4037 * Local context variables updated. 4038 * 4039 * 4040 * The routine sends packets with MLX5_OPCODE_EMPW 4041 * without inlining, this is dedicated optimized branch. 4042 * No VLAN insertion is supported. 4043 */ 4044 static __rte_always_inline enum mlx5_txcmp_code 4045 mlx5_tx_burst_empw_simple(struct mlx5_txq_data *restrict txq, 4046 struct rte_mbuf **restrict pkts, 4047 unsigned int pkts_n, 4048 struct mlx5_txq_local *restrict loc, 4049 unsigned int olx) 4050 { 4051 /* 4052 * Subroutine is the part of mlx5_tx_burst_single() 4053 * and sends single-segment packet with eMPW opcode 4054 * without data inlining. 4055 */ 4056 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE)); 4057 MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW)); 4058 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4059 MLX5_ASSERT(pkts_n > loc->pkts_sent); 4060 static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size"); 4061 pkts += loc->pkts_sent + 1; 4062 pkts_n -= loc->pkts_sent; 4063 for (;;) { 4064 struct mlx5_wqe_dseg *restrict dseg; 4065 struct mlx5_wqe_eseg *restrict eseg; 4066 enum mlx5_txcmp_code ret; 4067 unsigned int part, loop; 4068 unsigned int slen = 0; 4069 4070 next_empw: 4071 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4072 part = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ? 4073 MLX5_MPW_MAX_PACKETS : 4074 MLX5_EMPW_MAX_PACKETS); 4075 if (unlikely(loc->elts_free < part)) { 4076 /* We have no enough elts to save all mbufs. */ 4077 if (unlikely(loc->elts_free < MLX5_EMPW_MIN_PACKETS)) 4078 return MLX5_TXCMP_CODE_EXIT; 4079 /* But we still able to send at least minimal eMPW. */ 4080 part = loc->elts_free; 4081 } 4082 /* Check whether we have enough WQEs */ 4083 if (unlikely(loc->wqe_free < ((2 + part + 3) / 4))) { 4084 if (unlikely(loc->wqe_free < 4085 ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4))) 4086 return MLX5_TXCMP_CODE_EXIT; 4087 part = (loc->wqe_free * 4) - 2; 4088 } 4089 if (likely(part > 1)) 4090 rte_prefetch0(*pkts); 4091 loc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4092 /* 4093 * Build eMPW title WQEBB: 4094 * - Control Segment, eMPW opcode 4095 * - Ethernet Segment, no inline 4096 */ 4097 mlx5_tx_cseg_init(txq, loc, loc->wqe_last, part + 2, 4098 MLX5_OPCODE_ENHANCED_MPSW, olx); 4099 mlx5_tx_eseg_none(txq, loc, loc->wqe_last, 4100 olx & ~MLX5_TXOFF_CONFIG_VLAN); 4101 eseg = &loc->wqe_last->eseg; 4102 dseg = &loc->wqe_last->dseg[0]; 4103 loop = part; 4104 /* Store the packet length for legacy MPW. */ 4105 if (MLX5_TXOFF_CONFIG(MPW)) 4106 eseg->mss = rte_cpu_to_be_16 4107 (rte_pktmbuf_data_len(loc->mbuf)); 4108 for (;;) { 4109 uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf); 4110 #ifdef MLX5_PMD_SOFT_COUNTERS 4111 /* Update sent data bytes counter. */ 4112 slen += dlen; 4113 #endif 4114 mlx5_tx_dseg_ptr 4115 (txq, loc, dseg, 4116 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 4117 dlen, olx); 4118 if (unlikely(--loop == 0)) 4119 break; 4120 loc->mbuf = *pkts++; 4121 if (likely(loop > 1)) 4122 rte_prefetch0(*pkts); 4123 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4124 /* 4125 * Unroll the completion code to avoid 4126 * returning variable value - it results in 4127 * unoptimized sequent checking in caller. 4128 */ 4129 if (ret == MLX5_TXCMP_CODE_MULTI) { 4130 part -= loop; 4131 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4132 if (unlikely(!loc->elts_free || 4133 !loc->wqe_free)) 4134 return MLX5_TXCMP_CODE_EXIT; 4135 return MLX5_TXCMP_CODE_MULTI; 4136 } 4137 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4138 if (ret == MLX5_TXCMP_CODE_TSO) { 4139 part -= loop; 4140 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4141 if (unlikely(!loc->elts_free || 4142 !loc->wqe_free)) 4143 return MLX5_TXCMP_CODE_EXIT; 4144 return MLX5_TXCMP_CODE_TSO; 4145 } 4146 if (ret == MLX5_TXCMP_CODE_SINGLE) { 4147 part -= loop; 4148 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4149 if (unlikely(!loc->elts_free || 4150 !loc->wqe_free)) 4151 return MLX5_TXCMP_CODE_EXIT; 4152 return MLX5_TXCMP_CODE_SINGLE; 4153 } 4154 if (ret != MLX5_TXCMP_CODE_EMPW) { 4155 MLX5_ASSERT(false); 4156 part -= loop; 4157 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4158 return MLX5_TXCMP_CODE_ERROR; 4159 } 4160 /* 4161 * Check whether packet parameters coincide 4162 * within assumed eMPW batch: 4163 * - check sum settings 4164 * - metadata value 4165 * - software parser settings 4166 * - packets length (legacy MPW only) 4167 */ 4168 if (!mlx5_tx_match_empw(txq, eseg, loc, dlen, olx)) { 4169 MLX5_ASSERT(loop); 4170 part -= loop; 4171 mlx5_tx_sdone_empw(txq, loc, part, slen, olx); 4172 if (unlikely(!loc->elts_free || 4173 !loc->wqe_free)) 4174 return MLX5_TXCMP_CODE_EXIT; 4175 pkts_n -= part; 4176 goto next_empw; 4177 } 4178 /* Packet attributes match, continue the same eMPW. */ 4179 ++dseg; 4180 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 4181 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 4182 } 4183 /* eMPW is built successfully, update loop parameters. */ 4184 MLX5_ASSERT(!loop); 4185 MLX5_ASSERT(pkts_n >= part); 4186 #ifdef MLX5_PMD_SOFT_COUNTERS 4187 /* Update sent data bytes counter. */ 4188 txq->stats.obytes += slen; 4189 #endif 4190 loc->elts_free -= part; 4191 loc->pkts_sent += part; 4192 txq->wqe_ci += (2 + part + 3) / 4; 4193 loc->wqe_free -= (2 + part + 3) / 4; 4194 pkts_n -= part; 4195 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 4196 return MLX5_TXCMP_CODE_EXIT; 4197 loc->mbuf = *pkts++; 4198 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4199 if (unlikely(ret != MLX5_TXCMP_CODE_EMPW)) 4200 return ret; 4201 /* Continue sending eMPW batches. */ 4202 } 4203 MLX5_ASSERT(false); 4204 } 4205 4206 /** 4207 * The routine sends packets with MLX5_OPCODE_EMPW 4208 * with inlining, optionally supports VLAN insertion. 4209 */ 4210 static __rte_always_inline enum mlx5_txcmp_code 4211 mlx5_tx_burst_empw_inline(struct mlx5_txq_data *restrict txq, 4212 struct rte_mbuf **restrict pkts, 4213 unsigned int pkts_n, 4214 struct mlx5_txq_local *restrict loc, 4215 unsigned int olx) 4216 { 4217 /* 4218 * Subroutine is the part of mlx5_tx_burst_single() 4219 * and sends single-segment packet with eMPW opcode 4220 * with data inlining. 4221 */ 4222 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4223 MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW)); 4224 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4225 MLX5_ASSERT(pkts_n > loc->pkts_sent); 4226 static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size"); 4227 pkts += loc->pkts_sent + 1; 4228 pkts_n -= loc->pkts_sent; 4229 for (;;) { 4230 struct mlx5_wqe_dseg *restrict dseg; 4231 struct mlx5_wqe *restrict wqem; 4232 enum mlx5_txcmp_code ret; 4233 unsigned int room, part, nlim; 4234 unsigned int slen = 0; 4235 4236 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4237 /* 4238 * Limits the amount of packets in one WQE 4239 * to improve CQE latency generation. 4240 */ 4241 nlim = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ? 4242 MLX5_MPW_INLINE_MAX_PACKETS : 4243 MLX5_EMPW_MAX_PACKETS); 4244 /* Check whether we have minimal amount WQEs */ 4245 if (unlikely(loc->wqe_free < 4246 ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4))) 4247 return MLX5_TXCMP_CODE_EXIT; 4248 if (likely(pkts_n > 1)) 4249 rte_prefetch0(*pkts); 4250 wqem = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4251 /* 4252 * Build eMPW title WQEBB: 4253 * - Control Segment, eMPW opcode, zero DS 4254 * - Ethernet Segment, no inline 4255 */ 4256 mlx5_tx_cseg_init(txq, loc, wqem, 0, 4257 MLX5_OPCODE_ENHANCED_MPSW, olx); 4258 mlx5_tx_eseg_none(txq, loc, wqem, 4259 olx & ~MLX5_TXOFF_CONFIG_VLAN); 4260 dseg = &wqem->dseg[0]; 4261 /* Store the packet length for legacy MPW. */ 4262 if (MLX5_TXOFF_CONFIG(MPW)) 4263 wqem->eseg.mss = rte_cpu_to_be_16 4264 (rte_pktmbuf_data_len(loc->mbuf)); 4265 room = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE, 4266 loc->wqe_free) * MLX5_WQE_SIZE - 4267 MLX5_WQE_CSEG_SIZE - 4268 MLX5_WQE_ESEG_SIZE; 4269 /* Limit the room for legacy MPW sessions for performance. */ 4270 if (MLX5_TXOFF_CONFIG(MPW)) 4271 room = RTE_MIN(room, 4272 RTE_MAX(txq->inlen_empw + 4273 sizeof(dseg->bcount) + 4274 (MLX5_TXOFF_CONFIG(VLAN) ? 4275 sizeof(struct rte_vlan_hdr) : 0), 4276 MLX5_MPW_INLINE_MAX_PACKETS * 4277 MLX5_WQE_DSEG_SIZE)); 4278 /* Build WQE till we have space, packets and resources. */ 4279 part = room; 4280 for (;;) { 4281 uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf); 4282 uint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *); 4283 unsigned int tlen; 4284 4285 MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE); 4286 MLX5_ASSERT((room % MLX5_WQE_DSEG_SIZE) == 0); 4287 MLX5_ASSERT((uintptr_t)dseg < (uintptr_t)txq->wqes_end); 4288 /* 4289 * Some Tx offloads may cause an error if 4290 * packet is not long enough, check against 4291 * assumed minimal length. 4292 */ 4293 if (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) { 4294 part -= room; 4295 if (unlikely(!part)) 4296 return MLX5_TXCMP_CODE_ERROR; 4297 /* 4298 * We have some successfully built 4299 * packet Data Segments to send. 4300 */ 4301 mlx5_tx_idone_empw(txq, loc, part, 4302 slen, wqem, olx); 4303 return MLX5_TXCMP_CODE_ERROR; 4304 } 4305 /* Inline or not inline - that's the Question. */ 4306 if (dlen > txq->inlen_empw || 4307 loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) 4308 goto pointer_empw; 4309 if (MLX5_TXOFF_CONFIG(MPW)) { 4310 if (dlen > txq->inlen_send) 4311 goto pointer_empw; 4312 tlen = dlen; 4313 if (part == room) { 4314 /* Open new inline MPW session. */ 4315 tlen += sizeof(dseg->bcount); 4316 dseg->bcount = RTE_BE32(0); 4317 dseg = RTE_PTR_ADD 4318 (dseg, sizeof(dseg->bcount)); 4319 } else { 4320 /* 4321 * No pointer and inline descriptor 4322 * intermix for legacy MPW sessions. 4323 */ 4324 if (wqem->dseg[0].bcount) 4325 break; 4326 } 4327 } else { 4328 tlen = sizeof(dseg->bcount) + dlen; 4329 } 4330 /* Inline entire packet, optional VLAN insertion. */ 4331 if (MLX5_TXOFF_CONFIG(VLAN) && 4332 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 4333 /* 4334 * The packet length must be checked in 4335 * mlx5_tx_able_to_empw() and packet 4336 * fits into inline length guaranteed. 4337 */ 4338 MLX5_ASSERT((dlen + 4339 sizeof(struct rte_vlan_hdr)) <= 4340 txq->inlen_empw); 4341 tlen += sizeof(struct rte_vlan_hdr); 4342 if (room < tlen) 4343 break; 4344 dseg = mlx5_tx_dseg_vlan(txq, loc, dseg, 4345 dptr, dlen, olx); 4346 #ifdef MLX5_PMD_SOFT_COUNTERS 4347 /* Update sent data bytes counter. */ 4348 slen += sizeof(struct rte_vlan_hdr); 4349 #endif 4350 } else { 4351 if (room < tlen) 4352 break; 4353 dseg = mlx5_tx_dseg_empw(txq, loc, dseg, 4354 dptr, dlen, olx); 4355 } 4356 if (!MLX5_TXOFF_CONFIG(MPW)) 4357 tlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE); 4358 MLX5_ASSERT(room >= tlen); 4359 room -= tlen; 4360 /* 4361 * Packet data are completely inlined, 4362 * free the packet immediately. 4363 */ 4364 rte_pktmbuf_free_seg(loc->mbuf); 4365 goto next_mbuf; 4366 pointer_empw: 4367 /* 4368 * No pointer and inline descriptor 4369 * intermix for legacy MPW sessions. 4370 */ 4371 if (MLX5_TXOFF_CONFIG(MPW) && 4372 part != room && 4373 wqem->dseg[0].bcount == RTE_BE32(0)) 4374 break; 4375 /* 4376 * Not inlinable VLAN packets are 4377 * proceeded outside of this routine. 4378 */ 4379 MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE); 4380 if (MLX5_TXOFF_CONFIG(VLAN)) 4381 MLX5_ASSERT(!(loc->mbuf->ol_flags & 4382 PKT_TX_VLAN_PKT)); 4383 mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx); 4384 /* We have to store mbuf in elts.*/ 4385 txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf; 4386 room -= MLX5_WQE_DSEG_SIZE; 4387 /* Ring buffer wraparound is checked at the loop end.*/ 4388 ++dseg; 4389 next_mbuf: 4390 #ifdef MLX5_PMD_SOFT_COUNTERS 4391 /* Update sent data bytes counter. */ 4392 slen += dlen; 4393 #endif 4394 loc->pkts_sent++; 4395 loc->elts_free--; 4396 pkts_n--; 4397 if (unlikely(!pkts_n || !loc->elts_free)) { 4398 /* 4399 * We have no resources/packets to 4400 * continue build descriptors. 4401 */ 4402 part -= room; 4403 mlx5_tx_idone_empw(txq, loc, part, 4404 slen, wqem, olx); 4405 return MLX5_TXCMP_CODE_EXIT; 4406 } 4407 loc->mbuf = *pkts++; 4408 if (likely(pkts_n > 1)) 4409 rte_prefetch0(*pkts); 4410 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4411 /* 4412 * Unroll the completion code to avoid 4413 * returning variable value - it results in 4414 * unoptimized sequent checking in caller. 4415 */ 4416 if (ret == MLX5_TXCMP_CODE_MULTI) { 4417 part -= room; 4418 mlx5_tx_idone_empw(txq, loc, part, 4419 slen, wqem, olx); 4420 if (unlikely(!loc->elts_free || 4421 !loc->wqe_free)) 4422 return MLX5_TXCMP_CODE_EXIT; 4423 return MLX5_TXCMP_CODE_MULTI; 4424 } 4425 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4426 if (ret == MLX5_TXCMP_CODE_TSO) { 4427 part -= room; 4428 mlx5_tx_idone_empw(txq, loc, part, 4429 slen, wqem, olx); 4430 if (unlikely(!loc->elts_free || 4431 !loc->wqe_free)) 4432 return MLX5_TXCMP_CODE_EXIT; 4433 return MLX5_TXCMP_CODE_TSO; 4434 } 4435 if (ret == MLX5_TXCMP_CODE_SINGLE) { 4436 part -= room; 4437 mlx5_tx_idone_empw(txq, loc, part, 4438 slen, wqem, olx); 4439 if (unlikely(!loc->elts_free || 4440 !loc->wqe_free)) 4441 return MLX5_TXCMP_CODE_EXIT; 4442 return MLX5_TXCMP_CODE_SINGLE; 4443 } 4444 if (ret != MLX5_TXCMP_CODE_EMPW) { 4445 MLX5_ASSERT(false); 4446 part -= room; 4447 mlx5_tx_idone_empw(txq, loc, part, 4448 slen, wqem, olx); 4449 return MLX5_TXCMP_CODE_ERROR; 4450 } 4451 /* Check if we have minimal room left. */ 4452 nlim--; 4453 if (unlikely(!nlim || room < MLX5_WQE_DSEG_SIZE)) 4454 break; 4455 /* 4456 * Check whether packet parameters coincide 4457 * within assumed eMPW batch: 4458 * - check sum settings 4459 * - metadata value 4460 * - software parser settings 4461 * - packets length (legacy MPW only) 4462 */ 4463 if (!mlx5_tx_match_empw(txq, &wqem->eseg, 4464 loc, dlen, olx)) 4465 break; 4466 /* Packet attributes match, continue the same eMPW. */ 4467 if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end) 4468 dseg = (struct mlx5_wqe_dseg *)txq->wqes; 4469 } 4470 /* 4471 * We get here to close an existing eMPW 4472 * session and start the new one. 4473 */ 4474 MLX5_ASSERT(pkts_n); 4475 part -= room; 4476 if (unlikely(!part)) 4477 return MLX5_TXCMP_CODE_EXIT; 4478 mlx5_tx_idone_empw(txq, loc, part, slen, wqem, olx); 4479 if (unlikely(!loc->elts_free || 4480 !loc->wqe_free)) 4481 return MLX5_TXCMP_CODE_EXIT; 4482 /* Continue the loop with new eMPW session. */ 4483 } 4484 MLX5_ASSERT(false); 4485 } 4486 4487 /** 4488 * The routine sends packets with ordinary MLX5_OPCODE_SEND. 4489 * Data inlining and VLAN insertion are supported. 4490 */ 4491 static __rte_always_inline enum mlx5_txcmp_code 4492 mlx5_tx_burst_single_send(struct mlx5_txq_data *restrict txq, 4493 struct rte_mbuf **restrict pkts, 4494 unsigned int pkts_n, 4495 struct mlx5_txq_local *restrict loc, 4496 unsigned int olx) 4497 { 4498 /* 4499 * Subroutine is the part of mlx5_tx_burst_single() 4500 * and sends single-segment packet with SEND opcode. 4501 */ 4502 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4503 MLX5_ASSERT(pkts_n > loc->pkts_sent); 4504 pkts += loc->pkts_sent + 1; 4505 pkts_n -= loc->pkts_sent; 4506 for (;;) { 4507 struct mlx5_wqe *restrict wqe; 4508 enum mlx5_txcmp_code ret; 4509 4510 MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1); 4511 if (MLX5_TXOFF_CONFIG(INLINE)) { 4512 unsigned int inlen, vlan = 0; 4513 4514 inlen = rte_pktmbuf_data_len(loc->mbuf); 4515 if (MLX5_TXOFF_CONFIG(VLAN) && 4516 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) { 4517 vlan = sizeof(struct rte_vlan_hdr); 4518 inlen += vlan; 4519 static_assert((sizeof(struct rte_vlan_hdr) + 4520 sizeof(struct rte_ether_hdr)) == 4521 MLX5_ESEG_MIN_INLINE_SIZE, 4522 "invalid min inline data size"); 4523 } 4524 /* 4525 * If inlining is enabled at configuration time 4526 * the limit must be not less than minimal size. 4527 * Otherwise we would do extra check for data 4528 * size to avoid crashes due to length overflow. 4529 */ 4530 MLX5_ASSERT(txq->inlen_send >= 4531 MLX5_ESEG_MIN_INLINE_SIZE); 4532 if (inlen <= txq->inlen_send) { 4533 unsigned int seg_n, wqe_n; 4534 4535 rte_prefetch0(rte_pktmbuf_mtod 4536 (loc->mbuf, uint8_t *)); 4537 /* Check against minimal length. */ 4538 if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE) 4539 return MLX5_TXCMP_CODE_ERROR; 4540 if (loc->mbuf->ol_flags & 4541 PKT_TX_DYNF_NOINLINE) { 4542 /* 4543 * The hint flag not to inline packet 4544 * data is set. Check whether we can 4545 * follow the hint. 4546 */ 4547 if ((!MLX5_TXOFF_CONFIG(EMPW) && 4548 txq->inlen_mode) || 4549 (MLX5_TXOFF_CONFIG(MPW) && 4550 txq->inlen_mode)) { 4551 /* 4552 * The hardware requires the 4553 * minimal inline data header. 4554 */ 4555 goto single_min_inline; 4556 } 4557 if (MLX5_TXOFF_CONFIG(VLAN) && 4558 vlan && !txq->vlan_en) { 4559 /* 4560 * We must insert VLAN tag 4561 * by software means. 4562 */ 4563 goto single_part_inline; 4564 } 4565 goto single_no_inline; 4566 } 4567 /* 4568 * Completely inlined packet data WQE: 4569 * - Control Segment, SEND opcode 4570 * - Ethernet Segment, no VLAN insertion 4571 * - Data inlined, VLAN optionally inserted 4572 * - Alignment to MLX5_WSEG_SIZE 4573 * Have to estimate amount of WQEBBs 4574 */ 4575 seg_n = (inlen + 3 * MLX5_WSEG_SIZE - 4576 MLX5_ESEG_MIN_INLINE_SIZE + 4577 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 4578 /* Check if there are enough WQEBBs. */ 4579 wqe_n = (seg_n + 3) / 4; 4580 if (wqe_n > loc->wqe_free) 4581 return MLX5_TXCMP_CODE_EXIT; 4582 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4583 loc->wqe_last = wqe; 4584 mlx5_tx_cseg_init(txq, loc, wqe, seg_n, 4585 MLX5_OPCODE_SEND, olx); 4586 mlx5_tx_eseg_data(txq, loc, wqe, 4587 vlan, inlen, 0, olx); 4588 txq->wqe_ci += wqe_n; 4589 loc->wqe_free -= wqe_n; 4590 /* 4591 * Packet data are completely inlined, 4592 * free the packet immediately. 4593 */ 4594 rte_pktmbuf_free_seg(loc->mbuf); 4595 } else if ((!MLX5_TXOFF_CONFIG(EMPW) || 4596 MLX5_TXOFF_CONFIG(MPW)) && 4597 txq->inlen_mode) { 4598 /* 4599 * If minimal inlining is requested the eMPW 4600 * feature should be disabled due to data is 4601 * inlined into Ethernet Segment, which can 4602 * not contain inlined data for eMPW due to 4603 * segment shared for all packets. 4604 */ 4605 struct mlx5_wqe_dseg *restrict dseg; 4606 unsigned int ds; 4607 uint8_t *dptr; 4608 4609 /* 4610 * The inline-mode settings require 4611 * to inline the specified amount of 4612 * data bytes to the Ethernet Segment. 4613 * We should check the free space in 4614 * WQE ring buffer to inline partially. 4615 */ 4616 single_min_inline: 4617 MLX5_ASSERT(txq->inlen_send >= txq->inlen_mode); 4618 MLX5_ASSERT(inlen > txq->inlen_mode); 4619 MLX5_ASSERT(txq->inlen_mode >= 4620 MLX5_ESEG_MIN_INLINE_SIZE); 4621 /* 4622 * Check whether there are enough free WQEBBs: 4623 * - Control Segment 4624 * - Ethernet Segment 4625 * - First Segment of inlined Ethernet data 4626 * - ... data continued ... 4627 * - Finishing Data Segment of pointer type 4628 */ 4629 ds = (MLX5_WQE_CSEG_SIZE + 4630 MLX5_WQE_ESEG_SIZE + 4631 MLX5_WQE_DSEG_SIZE + 4632 txq->inlen_mode - 4633 MLX5_ESEG_MIN_INLINE_SIZE + 4634 MLX5_WQE_DSEG_SIZE + 4635 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE; 4636 if (loc->wqe_free < ((ds + 3) / 4)) 4637 return MLX5_TXCMP_CODE_EXIT; 4638 /* 4639 * Build the ordinary SEND WQE: 4640 * - Control Segment 4641 * - Ethernet Segment, inline inlen_mode bytes 4642 * - Data Segment of pointer type 4643 */ 4644 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4645 loc->wqe_last = wqe; 4646 mlx5_tx_cseg_init(txq, loc, wqe, ds, 4647 MLX5_OPCODE_SEND, olx); 4648 dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, 4649 txq->inlen_mode, 4650 0, olx); 4651 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + 4652 txq->inlen_mode - vlan; 4653 inlen -= txq->inlen_mode; 4654 mlx5_tx_dseg_ptr(txq, loc, dseg, 4655 dptr, inlen, olx); 4656 /* 4657 * WQE is built, update the loop parameters 4658 * and got to the next packet. 4659 */ 4660 txq->wqe_ci += (ds + 3) / 4; 4661 loc->wqe_free -= (ds + 3) / 4; 4662 /* We have to store mbuf in elts.*/ 4663 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4664 txq->elts[txq->elts_head++ & txq->elts_m] = 4665 loc->mbuf; 4666 --loc->elts_free; 4667 } else { 4668 uint8_t *dptr; 4669 unsigned int dlen; 4670 4671 /* 4672 * Partially inlined packet data WQE, we have 4673 * some space in title WQEBB, we can fill it 4674 * with some packet data. It takes one WQEBB, 4675 * it is available, no extra space check: 4676 * - Control Segment, SEND opcode 4677 * - Ethernet Segment, no VLAN insertion 4678 * - MLX5_ESEG_MIN_INLINE_SIZE bytes of Data 4679 * - Data Segment, pointer type 4680 * 4681 * We also get here if VLAN insertion is not 4682 * supported by HW, the inline is enabled. 4683 */ 4684 single_part_inline: 4685 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4686 loc->wqe_last = wqe; 4687 mlx5_tx_cseg_init(txq, loc, wqe, 4, 4688 MLX5_OPCODE_SEND, olx); 4689 mlx5_tx_eseg_dmin(txq, loc, wqe, vlan, olx); 4690 dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + 4691 MLX5_ESEG_MIN_INLINE_SIZE - vlan; 4692 /* 4693 * The length check is performed above, by 4694 * comparing with txq->inlen_send. We should 4695 * not get overflow here. 4696 */ 4697 MLX5_ASSERT(inlen > MLX5_ESEG_MIN_INLINE_SIZE); 4698 dlen = inlen - MLX5_ESEG_MIN_INLINE_SIZE; 4699 mlx5_tx_dseg_ptr(txq, loc, &wqe->dseg[1], 4700 dptr, dlen, olx); 4701 ++txq->wqe_ci; 4702 --loc->wqe_free; 4703 /* We have to store mbuf in elts.*/ 4704 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE)); 4705 txq->elts[txq->elts_head++ & txq->elts_m] = 4706 loc->mbuf; 4707 --loc->elts_free; 4708 } 4709 #ifdef MLX5_PMD_SOFT_COUNTERS 4710 /* Update sent data bytes counter. */ 4711 txq->stats.obytes += vlan + 4712 rte_pktmbuf_data_len(loc->mbuf); 4713 #endif 4714 } else { 4715 /* 4716 * No inline at all, it means the CPU cycles saving 4717 * is prioritized at configuration, we should not 4718 * copy any packet data to WQE. 4719 * 4720 * SEND WQE, one WQEBB: 4721 * - Control Segment, SEND opcode 4722 * - Ethernet Segment, optional VLAN, no inline 4723 * - Data Segment, pointer type 4724 */ 4725 single_no_inline: 4726 wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m); 4727 loc->wqe_last = wqe; 4728 mlx5_tx_cseg_init(txq, loc, wqe, 3, 4729 MLX5_OPCODE_SEND, olx); 4730 mlx5_tx_eseg_none(txq, loc, wqe, olx); 4731 mlx5_tx_dseg_ptr 4732 (txq, loc, &wqe->dseg[0], 4733 rte_pktmbuf_mtod(loc->mbuf, uint8_t *), 4734 rte_pktmbuf_data_len(loc->mbuf), olx); 4735 ++txq->wqe_ci; 4736 --loc->wqe_free; 4737 /* 4738 * We should not store mbuf pointer in elts 4739 * if no inlining is configured, this is done 4740 * by calling routine in a batch copy. 4741 */ 4742 MLX5_ASSERT(!MLX5_TXOFF_CONFIG(INLINE)); 4743 --loc->elts_free; 4744 #ifdef MLX5_PMD_SOFT_COUNTERS 4745 /* Update sent data bytes counter. */ 4746 txq->stats.obytes += rte_pktmbuf_data_len(loc->mbuf); 4747 if (MLX5_TXOFF_CONFIG(VLAN) && 4748 loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) 4749 txq->stats.obytes += 4750 sizeof(struct rte_vlan_hdr); 4751 #endif 4752 } 4753 ++loc->pkts_sent; 4754 --pkts_n; 4755 if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free)) 4756 return MLX5_TXCMP_CODE_EXIT; 4757 loc->mbuf = *pkts++; 4758 if (pkts_n > 1) 4759 rte_prefetch0(*pkts); 4760 ret = mlx5_tx_able_to_empw(txq, loc, olx, true); 4761 if (unlikely(ret != MLX5_TXCMP_CODE_SINGLE)) 4762 return ret; 4763 } 4764 MLX5_ASSERT(false); 4765 } 4766 4767 static __rte_always_inline enum mlx5_txcmp_code 4768 mlx5_tx_burst_single(struct mlx5_txq_data *restrict txq, 4769 struct rte_mbuf **restrict pkts, 4770 unsigned int pkts_n, 4771 struct mlx5_txq_local *restrict loc, 4772 unsigned int olx) 4773 { 4774 enum mlx5_txcmp_code ret; 4775 4776 ret = mlx5_tx_able_to_empw(txq, loc, olx, false); 4777 if (ret == MLX5_TXCMP_CODE_SINGLE) 4778 goto ordinary_send; 4779 MLX5_ASSERT(ret == MLX5_TXCMP_CODE_EMPW); 4780 for (;;) { 4781 /* Optimize for inline/no inline eMPW send. */ 4782 ret = (MLX5_TXOFF_CONFIG(INLINE)) ? 4783 mlx5_tx_burst_empw_inline 4784 (txq, pkts, pkts_n, loc, olx) : 4785 mlx5_tx_burst_empw_simple 4786 (txq, pkts, pkts_n, loc, olx); 4787 if (ret != MLX5_TXCMP_CODE_SINGLE) 4788 return ret; 4789 /* The resources to send one packet should remain. */ 4790 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4791 ordinary_send: 4792 ret = mlx5_tx_burst_single_send(txq, pkts, pkts_n, loc, olx); 4793 MLX5_ASSERT(ret != MLX5_TXCMP_CODE_SINGLE); 4794 if (ret != MLX5_TXCMP_CODE_EMPW) 4795 return ret; 4796 /* The resources to send one packet should remain. */ 4797 MLX5_ASSERT(loc->elts_free && loc->wqe_free); 4798 } 4799 } 4800 4801 /** 4802 * DPDK Tx callback template. This is configured template 4803 * used to generate routines optimized for specified offload setup. 4804 * One of this generated functions is chosen at SQ configuration 4805 * time. 4806 * 4807 * @param txq 4808 * Generic pointer to TX queue structure. 4809 * @param[in] pkts 4810 * Packets to transmit. 4811 * @param pkts_n 4812 * Number of packets in array. 4813 * @param olx 4814 * Configured offloads mask, presents the bits of MLX5_TXOFF_CONFIG_xxx 4815 * values. Should be static to take compile time static configuration 4816 * advantages. 4817 * 4818 * @return 4819 * Number of packets successfully transmitted (<= pkts_n). 4820 */ 4821 static __rte_always_inline uint16_t 4822 mlx5_tx_burst_tmpl(struct mlx5_txq_data *restrict txq, 4823 struct rte_mbuf **restrict pkts, 4824 uint16_t pkts_n, 4825 unsigned int olx) 4826 { 4827 struct mlx5_txq_local loc; 4828 enum mlx5_txcmp_code ret; 4829 unsigned int part; 4830 4831 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail)); 4832 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi)); 4833 if (unlikely(!pkts_n)) 4834 return 0; 4835 loc.pkts_sent = 0; 4836 loc.pkts_copy = 0; 4837 loc.wqe_last = NULL; 4838 4839 send_loop: 4840 loc.pkts_loop = loc.pkts_sent; 4841 /* 4842 * Check if there are some CQEs, if any: 4843 * - process an encountered errors 4844 * - process the completed WQEs 4845 * - free related mbufs 4846 * - doorbell the NIC about processed CQEs 4847 */ 4848 rte_prefetch0(*(pkts + loc.pkts_sent)); 4849 mlx5_tx_handle_completion(txq, olx); 4850 /* 4851 * Calculate the number of available resources - elts and WQEs. 4852 * There are two possible different scenarios: 4853 * - no data inlining into WQEs, one WQEBB may contains up to 4854 * four packets, in this case elts become scarce resource 4855 * - data inlining into WQEs, one packet may require multiple 4856 * WQEBBs, the WQEs become the limiting factor. 4857 */ 4858 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail)); 4859 loc.elts_free = txq->elts_s - 4860 (uint16_t)(txq->elts_head - txq->elts_tail); 4861 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi)); 4862 loc.wqe_free = txq->wqe_s - 4863 (uint16_t)(txq->wqe_ci - txq->wqe_pi); 4864 if (unlikely(!loc.elts_free || !loc.wqe_free)) 4865 goto burst_exit; 4866 for (;;) { 4867 /* 4868 * Fetch the packet from array. Usually this is 4869 * the first packet in series of multi/single 4870 * segment packets. 4871 */ 4872 loc.mbuf = *(pkts + loc.pkts_sent); 4873 /* Dedicated branch for multi-segment packets. */ 4874 if (MLX5_TXOFF_CONFIG(MULTI) && 4875 unlikely(NB_SEGS(loc.mbuf) > 1)) { 4876 /* 4877 * Multi-segment packet encountered. 4878 * Hardware is able to process it only 4879 * with SEND/TSO opcodes, one packet 4880 * per WQE, do it in dedicated routine. 4881 */ 4882 enter_send_multi: 4883 MLX5_ASSERT(loc.pkts_sent >= loc.pkts_copy); 4884 part = loc.pkts_sent - loc.pkts_copy; 4885 if (!MLX5_TXOFF_CONFIG(INLINE) && part) { 4886 /* 4887 * There are some single-segment mbufs not 4888 * stored in elts. The mbufs must be in the 4889 * same order as WQEs, so we must copy the 4890 * mbufs to elts here, before the coming 4891 * multi-segment packet mbufs is appended. 4892 */ 4893 mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, 4894 part, olx); 4895 loc.pkts_copy = loc.pkts_sent; 4896 } 4897 MLX5_ASSERT(pkts_n > loc.pkts_sent); 4898 ret = mlx5_tx_burst_mseg(txq, pkts, pkts_n, &loc, olx); 4899 if (!MLX5_TXOFF_CONFIG(INLINE)) 4900 loc.pkts_copy = loc.pkts_sent; 4901 /* 4902 * These returned code checks are supposed 4903 * to be optimized out due to routine inlining. 4904 */ 4905 if (ret == MLX5_TXCMP_CODE_EXIT) { 4906 /* 4907 * The routine returns this code when 4908 * all packets are sent or there is no 4909 * enough resources to complete request. 4910 */ 4911 break; 4912 } 4913 if (ret == MLX5_TXCMP_CODE_ERROR) { 4914 /* 4915 * The routine returns this code when 4916 * some error in the incoming packets 4917 * format occurred. 4918 */ 4919 txq->stats.oerrors++; 4920 break; 4921 } 4922 if (ret == MLX5_TXCMP_CODE_SINGLE) { 4923 /* 4924 * The single-segment packet was encountered 4925 * in the array, try to send it with the 4926 * best optimized way, possible engaging eMPW. 4927 */ 4928 goto enter_send_single; 4929 } 4930 if (MLX5_TXOFF_CONFIG(TSO) && 4931 ret == MLX5_TXCMP_CODE_TSO) { 4932 /* 4933 * The single-segment TSO packet was 4934 * encountered in the array. 4935 */ 4936 goto enter_send_tso; 4937 } 4938 /* We must not get here. Something is going wrong. */ 4939 MLX5_ASSERT(false); 4940 txq->stats.oerrors++; 4941 break; 4942 } 4943 /* Dedicated branch for single-segment TSO packets. */ 4944 if (MLX5_TXOFF_CONFIG(TSO) && 4945 unlikely(loc.mbuf->ol_flags & PKT_TX_TCP_SEG)) { 4946 /* 4947 * TSO might require special way for inlining 4948 * (dedicated parameters) and is sent with 4949 * MLX5_OPCODE_TSO opcode only, provide this 4950 * in dedicated branch. 4951 */ 4952 enter_send_tso: 4953 MLX5_ASSERT(NB_SEGS(loc.mbuf) == 1); 4954 MLX5_ASSERT(pkts_n > loc.pkts_sent); 4955 ret = mlx5_tx_burst_tso(txq, pkts, pkts_n, &loc, olx); 4956 /* 4957 * These returned code checks are supposed 4958 * to be optimized out due to routine inlining. 4959 */ 4960 if (ret == MLX5_TXCMP_CODE_EXIT) 4961 break; 4962 if (ret == MLX5_TXCMP_CODE_ERROR) { 4963 txq->stats.oerrors++; 4964 break; 4965 } 4966 if (ret == MLX5_TXCMP_CODE_SINGLE) 4967 goto enter_send_single; 4968 if (MLX5_TXOFF_CONFIG(MULTI) && 4969 ret == MLX5_TXCMP_CODE_MULTI) { 4970 /* 4971 * The multi-segment packet was 4972 * encountered in the array. 4973 */ 4974 goto enter_send_multi; 4975 } 4976 /* We must not get here. Something is going wrong. */ 4977 MLX5_ASSERT(false); 4978 txq->stats.oerrors++; 4979 break; 4980 } 4981 /* 4982 * The dedicated branch for the single-segment packets 4983 * without TSO. Often these ones can be sent using 4984 * MLX5_OPCODE_EMPW with multiple packets in one WQE. 4985 * The routine builds the WQEs till it encounters 4986 * the TSO or multi-segment packet (in case if these 4987 * offloads are requested at SQ configuration time). 4988 */ 4989 enter_send_single: 4990 MLX5_ASSERT(pkts_n > loc.pkts_sent); 4991 ret = mlx5_tx_burst_single(txq, pkts, pkts_n, &loc, olx); 4992 /* 4993 * These returned code checks are supposed 4994 * to be optimized out due to routine inlining. 4995 */ 4996 if (ret == MLX5_TXCMP_CODE_EXIT) 4997 break; 4998 if (ret == MLX5_TXCMP_CODE_ERROR) { 4999 txq->stats.oerrors++; 5000 break; 5001 } 5002 if (MLX5_TXOFF_CONFIG(MULTI) && 5003 ret == MLX5_TXCMP_CODE_MULTI) { 5004 /* 5005 * The multi-segment packet was 5006 * encountered in the array. 5007 */ 5008 goto enter_send_multi; 5009 } 5010 if (MLX5_TXOFF_CONFIG(TSO) && 5011 ret == MLX5_TXCMP_CODE_TSO) { 5012 /* 5013 * The single-segment TSO packet was 5014 * encountered in the array. 5015 */ 5016 goto enter_send_tso; 5017 } 5018 /* We must not get here. Something is going wrong. */ 5019 MLX5_ASSERT(false); 5020 txq->stats.oerrors++; 5021 break; 5022 } 5023 /* 5024 * Main Tx loop is completed, do the rest: 5025 * - set completion request if thresholds are reached 5026 * - doorbell the hardware 5027 * - copy the rest of mbufs to elts (if any) 5028 */ 5029 MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE) || 5030 loc.pkts_sent >= loc.pkts_copy); 5031 /* Take a shortcut if nothing is sent. */ 5032 if (unlikely(loc.pkts_sent == loc.pkts_loop)) 5033 goto burst_exit; 5034 /* Request CQE generation if limits are reached. */ 5035 mlx5_tx_request_completion(txq, &loc, olx); 5036 /* 5037 * Ring QP doorbell immediately after WQE building completion 5038 * to improve latencies. The pure software related data treatment 5039 * can be completed after doorbell. Tx CQEs for this SQ are 5040 * processed in this thread only by the polling. 5041 * 5042 * The rdma core library can map doorbell register in two ways, 5043 * depending on the environment variable "MLX5_SHUT_UP_BF": 5044 * 5045 * - as regular cached memory, the variable is either missing or 5046 * set to zero. This type of mapping may cause the significant 5047 * doorbell register writing latency and requires explicit 5048 * memory write barrier to mitigate this issue and prevent 5049 * write combining. 5050 * 5051 * - as non-cached memory, the variable is present and set to 5052 * not "0" value. This type of mapping may cause performance 5053 * impact under heavy loading conditions but the explicit write 5054 * memory barrier is not required and it may improve core 5055 * performance. 5056 * 5057 * - the legacy behaviour (prior 19.08 release) was to use some 5058 * heuristics to decide whether write memory barrier should 5059 * be performed. This behavior is supported with specifying 5060 * tx_db_nc=2, write barrier is skipped if application 5061 * provides the full recommended burst of packets, it 5062 * supposes the next packets are coming and the write barrier 5063 * will be issued on the next burst (after descriptor writing, 5064 * at least). 5065 */ 5066 mlx5_tx_dbrec_cond_wmb(txq, loc.wqe_last, !txq->db_nc && 5067 (!txq->db_heu || pkts_n % MLX5_TX_DEFAULT_BURST)); 5068 /* Not all of the mbufs may be stored into elts yet. */ 5069 part = MLX5_TXOFF_CONFIG(INLINE) ? 0 : loc.pkts_sent - loc.pkts_copy; 5070 if (!MLX5_TXOFF_CONFIG(INLINE) && part) { 5071 /* 5072 * There are some single-segment mbufs not stored in elts. 5073 * It can be only if the last packet was single-segment. 5074 * The copying is gathered into one place due to it is 5075 * a good opportunity to optimize that with SIMD. 5076 * Unfortunately if inlining is enabled the gaps in 5077 * pointer array may happen due to early freeing of the 5078 * inlined mbufs. 5079 */ 5080 mlx5_tx_copy_elts(txq, pkts + loc.pkts_copy, part, olx); 5081 loc.pkts_copy = loc.pkts_sent; 5082 } 5083 MLX5_ASSERT(txq->elts_s >= (uint16_t)(txq->elts_head - txq->elts_tail)); 5084 MLX5_ASSERT(txq->wqe_s >= (uint16_t)(txq->wqe_ci - txq->wqe_pi)); 5085 if (pkts_n > loc.pkts_sent) { 5086 /* 5087 * If burst size is large there might be no enough CQE 5088 * fetched from completion queue and no enough resources 5089 * freed to send all the packets. 5090 */ 5091 goto send_loop; 5092 } 5093 burst_exit: 5094 #ifdef MLX5_PMD_SOFT_COUNTERS 5095 /* Increment sent packets counter. */ 5096 txq->stats.opackets += loc.pkts_sent; 5097 #endif 5098 return loc.pkts_sent; 5099 } 5100 5101 /* Generate routines with Enhanced Multi-Packet Write support. */ 5102 MLX5_TXOFF_DECL(full_empw, 5103 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_EMPW) 5104 5105 MLX5_TXOFF_DECL(none_empw, 5106 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW) 5107 5108 MLX5_TXOFF_DECL(md_empw, 5109 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5110 5111 MLX5_TXOFF_DECL(mt_empw, 5112 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5113 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5114 5115 MLX5_TXOFF_DECL(mtsc_empw, 5116 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5117 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5118 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5119 5120 MLX5_TXOFF_DECL(mti_empw, 5121 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5122 MLX5_TXOFF_CONFIG_INLINE | 5123 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5124 5125 MLX5_TXOFF_DECL(mtv_empw, 5126 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5127 MLX5_TXOFF_CONFIG_VLAN | 5128 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5129 5130 MLX5_TXOFF_DECL(mtiv_empw, 5131 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5132 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5133 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5134 5135 MLX5_TXOFF_DECL(sc_empw, 5136 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5137 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5138 5139 MLX5_TXOFF_DECL(sci_empw, 5140 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5141 MLX5_TXOFF_CONFIG_INLINE | 5142 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5143 5144 MLX5_TXOFF_DECL(scv_empw, 5145 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5146 MLX5_TXOFF_CONFIG_VLAN | 5147 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5148 5149 MLX5_TXOFF_DECL(sciv_empw, 5150 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5151 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5152 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5153 5154 MLX5_TXOFF_DECL(i_empw, 5155 MLX5_TXOFF_CONFIG_INLINE | 5156 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5157 5158 MLX5_TXOFF_DECL(v_empw, 5159 MLX5_TXOFF_CONFIG_VLAN | 5160 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5161 5162 MLX5_TXOFF_DECL(iv_empw, 5163 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5164 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5165 5166 /* Generate routines without Enhanced Multi-Packet Write support. */ 5167 MLX5_TXOFF_DECL(full, 5168 MLX5_TXOFF_CONFIG_FULL) 5169 5170 MLX5_TXOFF_DECL(none, 5171 MLX5_TXOFF_CONFIG_NONE) 5172 5173 MLX5_TXOFF_DECL(md, 5174 MLX5_TXOFF_CONFIG_METADATA) 5175 5176 MLX5_TXOFF_DECL(mt, 5177 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5178 MLX5_TXOFF_CONFIG_METADATA) 5179 5180 MLX5_TXOFF_DECL(mtsc, 5181 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5182 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5183 MLX5_TXOFF_CONFIG_METADATA) 5184 5185 MLX5_TXOFF_DECL(mti, 5186 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5187 MLX5_TXOFF_CONFIG_INLINE | 5188 MLX5_TXOFF_CONFIG_METADATA) 5189 5190 5191 MLX5_TXOFF_DECL(mtv, 5192 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5193 MLX5_TXOFF_CONFIG_VLAN | 5194 MLX5_TXOFF_CONFIG_METADATA) 5195 5196 5197 MLX5_TXOFF_DECL(mtiv, 5198 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5199 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5200 MLX5_TXOFF_CONFIG_METADATA) 5201 5202 MLX5_TXOFF_DECL(sc, 5203 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5204 MLX5_TXOFF_CONFIG_METADATA) 5205 5206 MLX5_TXOFF_DECL(sci, 5207 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5208 MLX5_TXOFF_CONFIG_INLINE | 5209 MLX5_TXOFF_CONFIG_METADATA) 5210 5211 5212 MLX5_TXOFF_DECL(scv, 5213 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5214 MLX5_TXOFF_CONFIG_VLAN | 5215 MLX5_TXOFF_CONFIG_METADATA) 5216 5217 5218 MLX5_TXOFF_DECL(sciv, 5219 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5220 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5221 MLX5_TXOFF_CONFIG_METADATA) 5222 5223 MLX5_TXOFF_DECL(i, 5224 MLX5_TXOFF_CONFIG_INLINE | 5225 MLX5_TXOFF_CONFIG_METADATA) 5226 5227 MLX5_TXOFF_DECL(v, 5228 MLX5_TXOFF_CONFIG_VLAN | 5229 MLX5_TXOFF_CONFIG_METADATA) 5230 5231 MLX5_TXOFF_DECL(iv, 5232 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5233 MLX5_TXOFF_CONFIG_METADATA) 5234 5235 /* 5236 * Generate routines with Legacy Multi-Packet Write support. 5237 * This mode is supported by ConnectX-4 Lx only and imposes 5238 * offload limitations, not supported: 5239 * - ACL/Flows (metadata are becoming meaningless) 5240 * - WQE Inline headers 5241 * - SRIOV (E-Switch offloads) 5242 * - VLAN insertion 5243 * - tunnel encapsulation/decapsulation 5244 * - TSO 5245 */ 5246 MLX5_TXOFF_DECL(none_mpw, 5247 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW | 5248 MLX5_TXOFF_CONFIG_MPW) 5249 5250 MLX5_TXOFF_DECL(mci_mpw, 5251 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5252 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5253 MLX5_TXOFF_CONFIG_MPW) 5254 5255 MLX5_TXOFF_DECL(mc_mpw, 5256 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5257 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW) 5258 5259 MLX5_TXOFF_DECL(i_mpw, 5260 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5261 MLX5_TXOFF_CONFIG_MPW) 5262 5263 /* 5264 * Array of declared and compiled Tx burst function and corresponding 5265 * supported offloads set. The array is used to select the Tx burst 5266 * function for specified offloads set at Tx queue configuration time. 5267 */ 5268 const struct { 5269 eth_tx_burst_t func; 5270 unsigned int olx; 5271 } txoff_func[] = { 5272 MLX5_TXOFF_INFO(full_empw, 5273 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5274 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5275 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5276 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5277 5278 MLX5_TXOFF_INFO(none_empw, 5279 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW) 5280 5281 MLX5_TXOFF_INFO(md_empw, 5282 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5283 5284 MLX5_TXOFF_INFO(mt_empw, 5285 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5286 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5287 5288 MLX5_TXOFF_INFO(mtsc_empw, 5289 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5290 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5291 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5292 5293 MLX5_TXOFF_INFO(mti_empw, 5294 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5295 MLX5_TXOFF_CONFIG_INLINE | 5296 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5297 5298 MLX5_TXOFF_INFO(mtv_empw, 5299 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5300 MLX5_TXOFF_CONFIG_VLAN | 5301 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5302 5303 MLX5_TXOFF_INFO(mtiv_empw, 5304 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5305 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5306 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5307 5308 MLX5_TXOFF_INFO(sc_empw, 5309 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5310 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5311 5312 MLX5_TXOFF_INFO(sci_empw, 5313 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5314 MLX5_TXOFF_CONFIG_INLINE | 5315 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5316 5317 MLX5_TXOFF_INFO(scv_empw, 5318 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5319 MLX5_TXOFF_CONFIG_VLAN | 5320 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5321 5322 MLX5_TXOFF_INFO(sciv_empw, 5323 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5324 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5325 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5326 5327 MLX5_TXOFF_INFO(i_empw, 5328 MLX5_TXOFF_CONFIG_INLINE | 5329 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5330 5331 MLX5_TXOFF_INFO(v_empw, 5332 MLX5_TXOFF_CONFIG_VLAN | 5333 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5334 5335 MLX5_TXOFF_INFO(iv_empw, 5336 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5337 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 5338 5339 MLX5_TXOFF_INFO(full, 5340 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5341 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5342 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5343 MLX5_TXOFF_CONFIG_METADATA) 5344 5345 MLX5_TXOFF_INFO(none, 5346 MLX5_TXOFF_CONFIG_NONE) 5347 5348 MLX5_TXOFF_INFO(md, 5349 MLX5_TXOFF_CONFIG_METADATA) 5350 5351 MLX5_TXOFF_INFO(mt, 5352 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5353 MLX5_TXOFF_CONFIG_METADATA) 5354 5355 MLX5_TXOFF_INFO(mtsc, 5356 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5357 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5358 MLX5_TXOFF_CONFIG_METADATA) 5359 5360 MLX5_TXOFF_INFO(mti, 5361 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5362 MLX5_TXOFF_CONFIG_INLINE | 5363 MLX5_TXOFF_CONFIG_METADATA) 5364 5365 MLX5_TXOFF_INFO(mtv, 5366 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5367 MLX5_TXOFF_CONFIG_VLAN | 5368 MLX5_TXOFF_CONFIG_METADATA) 5369 5370 MLX5_TXOFF_INFO(mtiv, 5371 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 5372 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5373 MLX5_TXOFF_CONFIG_METADATA) 5374 5375 MLX5_TXOFF_INFO(sc, 5376 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5377 MLX5_TXOFF_CONFIG_METADATA) 5378 5379 MLX5_TXOFF_INFO(sci, 5380 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5381 MLX5_TXOFF_CONFIG_INLINE | 5382 MLX5_TXOFF_CONFIG_METADATA) 5383 5384 MLX5_TXOFF_INFO(scv, 5385 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5386 MLX5_TXOFF_CONFIG_VLAN | 5387 MLX5_TXOFF_CONFIG_METADATA) 5388 5389 MLX5_TXOFF_INFO(sciv, 5390 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 5391 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5392 MLX5_TXOFF_CONFIG_METADATA) 5393 5394 MLX5_TXOFF_INFO(i, 5395 MLX5_TXOFF_CONFIG_INLINE | 5396 MLX5_TXOFF_CONFIG_METADATA) 5397 5398 MLX5_TXOFF_INFO(v, 5399 MLX5_TXOFF_CONFIG_VLAN | 5400 MLX5_TXOFF_CONFIG_METADATA) 5401 5402 MLX5_TXOFF_INFO(iv, 5403 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 5404 MLX5_TXOFF_CONFIG_METADATA) 5405 5406 MLX5_TXOFF_INFO(none_mpw, 5407 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW | 5408 MLX5_TXOFF_CONFIG_MPW) 5409 5410 MLX5_TXOFF_INFO(mci_mpw, 5411 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5412 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5413 MLX5_TXOFF_CONFIG_MPW) 5414 5415 MLX5_TXOFF_INFO(mc_mpw, 5416 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 5417 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW) 5418 5419 MLX5_TXOFF_INFO(i_mpw, 5420 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 5421 MLX5_TXOFF_CONFIG_MPW) 5422 }; 5423 5424 /** 5425 * Configure the Tx function to use. The routine checks configured 5426 * Tx offloads for the device and selects appropriate Tx burst 5427 * routine. There are multiple Tx burst routines compiled from 5428 * the same template in the most optimal way for the dedicated 5429 * Tx offloads set. 5430 * 5431 * @param dev 5432 * Pointer to private data structure. 5433 * 5434 * @return 5435 * Pointer to selected Tx burst function. 5436 */ 5437 eth_tx_burst_t 5438 mlx5_select_tx_function(struct rte_eth_dev *dev) 5439 { 5440 struct mlx5_priv *priv = dev->data->dev_private; 5441 struct mlx5_dev_config *config = &priv->config; 5442 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; 5443 unsigned int diff = 0, olx = 0, i, m; 5444 5445 static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <= 5446 MLX5_DSEG_MAX, "invalid WQE max size"); 5447 static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE, 5448 "invalid WQE Control Segment size"); 5449 static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE, 5450 "invalid WQE Ethernet Segment size"); 5451 static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE, 5452 "invalid WQE Data Segment size"); 5453 static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE, 5454 "invalid WQE size"); 5455 MLX5_ASSERT(priv); 5456 if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) { 5457 /* We should support Multi-Segment Packets. */ 5458 olx |= MLX5_TXOFF_CONFIG_MULTI; 5459 } 5460 if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO | 5461 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 5462 DEV_TX_OFFLOAD_GRE_TNL_TSO | 5463 DEV_TX_OFFLOAD_IP_TNL_TSO | 5464 DEV_TX_OFFLOAD_UDP_TNL_TSO)) { 5465 /* We should support TCP Send Offload. */ 5466 olx |= MLX5_TXOFF_CONFIG_TSO; 5467 } 5468 if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO | 5469 DEV_TX_OFFLOAD_UDP_TNL_TSO | 5470 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { 5471 /* We should support Software Parser for Tunnels. */ 5472 olx |= MLX5_TXOFF_CONFIG_SWP; 5473 } 5474 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | 5475 DEV_TX_OFFLOAD_UDP_CKSUM | 5476 DEV_TX_OFFLOAD_TCP_CKSUM | 5477 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { 5478 /* We should support IP/TCP/UDP Checksums. */ 5479 olx |= MLX5_TXOFF_CONFIG_CSUM; 5480 } 5481 if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) { 5482 /* We should support VLAN insertion. */ 5483 olx |= MLX5_TXOFF_CONFIG_VLAN; 5484 } 5485 if (priv->txqs_n && (*priv->txqs)[0]) { 5486 struct mlx5_txq_data *txd = (*priv->txqs)[0]; 5487 5488 if (txd->inlen_send) { 5489 /* 5490 * Check the data inline requirements. Data inline 5491 * is enabled on per device basis, we can check 5492 * the first Tx queue only. 5493 * 5494 * If device does not support VLAN insertion in WQE 5495 * and some queues are requested to perform VLAN 5496 * insertion offload than inline must be enabled. 5497 */ 5498 olx |= MLX5_TXOFF_CONFIG_INLINE; 5499 } 5500 } 5501 if (config->mps == MLX5_MPW_ENHANCED && 5502 config->txq_inline_min <= 0) { 5503 /* 5504 * The NIC supports Enhanced Multi-Packet Write 5505 * and does not require minimal inline data. 5506 */ 5507 olx |= MLX5_TXOFF_CONFIG_EMPW; 5508 } 5509 if (rte_flow_dynf_metadata_avail()) { 5510 /* We should support Flow metadata. */ 5511 olx |= MLX5_TXOFF_CONFIG_METADATA; 5512 } 5513 if (config->mps == MLX5_MPW) { 5514 /* 5515 * The NIC supports Legacy Multi-Packet Write. 5516 * The MLX5_TXOFF_CONFIG_MPW controls the 5517 * descriptor building method in combination 5518 * with MLX5_TXOFF_CONFIG_EMPW. 5519 */ 5520 if (!(olx & (MLX5_TXOFF_CONFIG_TSO | 5521 MLX5_TXOFF_CONFIG_SWP | 5522 MLX5_TXOFF_CONFIG_VLAN | 5523 MLX5_TXOFF_CONFIG_METADATA))) 5524 olx |= MLX5_TXOFF_CONFIG_EMPW | 5525 MLX5_TXOFF_CONFIG_MPW; 5526 } 5527 /* 5528 * Scan the routines table to find the minimal 5529 * satisfying routine with requested offloads. 5530 */ 5531 m = RTE_DIM(txoff_func); 5532 for (i = 0; i < RTE_DIM(txoff_func); i++) { 5533 unsigned int tmp; 5534 5535 tmp = txoff_func[i].olx; 5536 if (tmp == olx) { 5537 /* Meets requested offloads exactly.*/ 5538 m = i; 5539 break; 5540 } 5541 if ((tmp & olx) != olx) { 5542 /* Does not meet requested offloads at all. */ 5543 continue; 5544 } 5545 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW) 5546 /* Do not enable eMPW if not configured. */ 5547 continue; 5548 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE) 5549 /* Do not enable inlining if not configured. */ 5550 continue; 5551 /* 5552 * Some routine meets the requirements. 5553 * Check whether it has minimal amount 5554 * of not requested offloads. 5555 */ 5556 tmp = __builtin_popcountl(tmp & ~olx); 5557 if (m >= RTE_DIM(txoff_func) || tmp < diff) { 5558 /* First or better match, save and continue. */ 5559 m = i; 5560 diff = tmp; 5561 continue; 5562 } 5563 if (tmp == diff) { 5564 tmp = txoff_func[i].olx ^ txoff_func[m].olx; 5565 if (__builtin_ffsl(txoff_func[i].olx & ~tmp) < 5566 __builtin_ffsl(txoff_func[m].olx & ~tmp)) { 5567 /* Lighter not requested offload. */ 5568 m = i; 5569 } 5570 } 5571 } 5572 if (m >= RTE_DIM(txoff_func)) { 5573 DRV_LOG(DEBUG, "port %u has no selected Tx function" 5574 " for requested offloads %04X", 5575 dev->data->port_id, olx); 5576 return NULL; 5577 } 5578 DRV_LOG(DEBUG, "port %u has selected Tx function" 5579 " supporting offloads %04X/%04X", 5580 dev->data->port_id, olx, txoff_func[m].olx); 5581 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI) 5582 DRV_LOG(DEBUG, "\tMULTI (multi segment)"); 5583 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO) 5584 DRV_LOG(DEBUG, "\tTSO (TCP send offload)"); 5585 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP) 5586 DRV_LOG(DEBUG, "\tSWP (software parser)"); 5587 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM) 5588 DRV_LOG(DEBUG, "\tCSUM (checksum offload)"); 5589 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE) 5590 DRV_LOG(DEBUG, "\tINLIN (inline data)"); 5591 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN) 5592 DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)"); 5593 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA) 5594 DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)"); 5595 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) { 5596 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW) 5597 DRV_LOG(DEBUG, "\tMPW (Legacy MPW)"); 5598 else 5599 DRV_LOG(DEBUG, "\tEMPW (Enhanced MPW)"); 5600 } 5601 return txoff_func[m].func; 5602 } 5603 5604 /** 5605 * DPDK callback to get the TX queue information 5606 * 5607 * @param dev 5608 * Pointer to the device structure. 5609 * 5610 * @param tx_queue_id 5611 * Tx queue identificator. 5612 * 5613 * @param qinfo 5614 * Pointer to the TX queue information structure. 5615 * 5616 * @return 5617 * None. 5618 */ 5619 5620 void 5621 mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, 5622 struct rte_eth_txq_info *qinfo) 5623 { 5624 struct mlx5_priv *priv = dev->data->dev_private; 5625 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id]; 5626 struct mlx5_txq_ctrl *txq_ctrl = 5627 container_of(txq, struct mlx5_txq_ctrl, txq); 5628 5629 if (!txq) 5630 return; 5631 qinfo->nb_desc = txq->elts_s; 5632 qinfo->conf.tx_thresh.pthresh = 0; 5633 qinfo->conf.tx_thresh.hthresh = 0; 5634 qinfo->conf.tx_thresh.wthresh = 0; 5635 qinfo->conf.tx_rs_thresh = 0; 5636 qinfo->conf.tx_free_thresh = 0; 5637 qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1; 5638 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 5639 } 5640 5641 /** 5642 * DPDK callback to get the TX packet burst mode information 5643 * 5644 * @param dev 5645 * Pointer to the device structure. 5646 * 5647 * @param tx_queue_id 5648 * Tx queue identificatior. 5649 * 5650 * @param mode 5651 * Pointer to the burts mode information. 5652 * 5653 * @return 5654 * 0 as success, -EINVAL as failure. 5655 */ 5656 5657 int 5658 mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, 5659 uint16_t tx_queue_id __rte_unused, 5660 struct rte_eth_burst_mode *mode) 5661 { 5662 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 5663 unsigned int i, olx; 5664 5665 for (i = 0; i < RTE_DIM(txoff_func); i++) { 5666 if (pkt_burst == txoff_func[i].func) { 5667 olx = txoff_func[i].olx; 5668 snprintf(mode->info, sizeof(mode->info), 5669 "%s%s%s%s%s%s%s%s", 5670 (olx & MLX5_TXOFF_CONFIG_EMPW) ? 5671 ((olx & MLX5_TXOFF_CONFIG_MPW) ? 5672 "Legacy MPW" : "Enhanced MPW") : "No MPW", 5673 (olx & MLX5_TXOFF_CONFIG_MULTI) ? 5674 " + MULTI" : "", 5675 (olx & MLX5_TXOFF_CONFIG_TSO) ? 5676 " + TSO" : "", 5677 (olx & MLX5_TXOFF_CONFIG_SWP) ? 5678 " + SWP" : "", 5679 (olx & MLX5_TXOFF_CONFIG_CSUM) ? 5680 " + CSUM" : "", 5681 (olx & MLX5_TXOFF_CONFIG_INLINE) ? 5682 " + INLINE" : "", 5683 (olx & MLX5_TXOFF_CONFIG_VLAN) ? 5684 " + VLAN" : "", 5685 (olx & MLX5_TXOFF_CONFIG_METADATA) ? 5686 " + METADATA" : ""); 5687 return 0; 5688 } 5689 } 5690 return -EINVAL; 5691 } 5692