1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef RTE_PMD_MLX5_RXTX_H_ 7 #define RTE_PMD_MLX5_RXTX_H_ 8 9 #include <stddef.h> 10 #include <stdint.h> 11 #include <sys/queue.h> 12 13 #include <rte_mbuf.h> 14 #include <rte_mempool.h> 15 #include <rte_common.h> 16 #include <rte_hexdump.h> 17 #include <rte_atomic.h> 18 #include <rte_spinlock.h> 19 #include <rte_io.h> 20 #include <rte_bus_pci.h> 21 #include <rte_malloc.h> 22 #include <rte_cycles.h> 23 24 #include <mlx5_glue.h> 25 #include <mlx5_prm.h> 26 #include <mlx5_common.h> 27 #include <mlx5_common_mr.h> 28 29 #include "mlx5_defs.h" 30 #include "mlx5_utils.h" 31 #include "mlx5.h" 32 #include "mlx5_autoconf.h" 33 34 /* Support tunnel matching. */ 35 #define MLX5_FLOW_TUNNEL 10 36 37 /* Mbuf dynamic flag offset for inline. */ 38 extern uint64_t rte_net_mlx5_dynf_inline_mask; 39 40 struct mlx5_rxq_stats { 41 #ifdef MLX5_PMD_SOFT_COUNTERS 42 uint64_t ipackets; /**< Total of successfully received packets. */ 43 uint64_t ibytes; /**< Total of successfully received bytes. */ 44 #endif 45 uint64_t idropped; /**< Total of packets dropped when RX ring full. */ 46 uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */ 47 }; 48 49 struct mlx5_txq_stats { 50 #ifdef MLX5_PMD_SOFT_COUNTERS 51 uint64_t opackets; /**< Total of successfully sent packets. */ 52 uint64_t obytes; /**< Total of successfully sent bytes. */ 53 #endif 54 uint64_t oerrors; /**< Total number of failed transmitted packets. */ 55 }; 56 57 struct mlx5_priv; 58 59 /* Compressed CQE context. */ 60 struct rxq_zip { 61 uint16_t ai; /* Array index. */ 62 uint16_t ca; /* Current array index. */ 63 uint16_t na; /* Next array index. */ 64 uint16_t cq_ci; /* The next CQE. */ 65 uint32_t cqe_cnt; /* Number of CQEs. */ 66 }; 67 68 /* Multi-Packet RQ buffer header. */ 69 struct mlx5_mprq_buf { 70 struct rte_mempool *mp; 71 rte_atomic16_t refcnt; /* Atomically accessed refcnt. */ 72 uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */ 73 struct rte_mbuf_ext_shared_info shinfos[]; 74 /* 75 * Shared information per stride. 76 * More memory will be allocated for the first stride head-room and for 77 * the strides data. 78 */ 79 } __rte_cache_aligned; 80 81 /* Get pointer to the first stride. */ 82 #define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \ 83 sizeof(struct mlx5_mprq_buf) + \ 84 (strd_n) * \ 85 sizeof(struct rte_mbuf_ext_shared_info) + \ 86 RTE_PKTMBUF_HEADROOM)) 87 88 #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6 89 #define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9 90 91 enum mlx5_rxq_err_state { 92 MLX5_RXQ_ERR_STATE_NO_ERROR = 0, 93 MLX5_RXQ_ERR_STATE_NEED_RESET, 94 MLX5_RXQ_ERR_STATE_NEED_READY, 95 }; 96 97 /* RX queue descriptor. */ 98 struct mlx5_rxq_data { 99 unsigned int csum:1; /* Enable checksum offloading. */ 100 unsigned int hw_timestamp:1; /* Enable HW timestamp. */ 101 unsigned int rt_timestamp:1; /* Realtime timestamp format. */ 102 unsigned int vlan_strip:1; /* Enable VLAN stripping. */ 103 unsigned int crc_present:1; /* CRC must be subtracted. */ 104 unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */ 105 unsigned int cqe_n:4; /* Log 2 of CQ elements. */ 106 unsigned int elts_n:4; /* Log 2 of Mbufs. */ 107 unsigned int rss_hash:1; /* RSS hash result is enabled. */ 108 unsigned int mark:1; /* Marked flow available on the queue. */ 109 unsigned int strd_num_n:5; /* Log 2 of the number of stride. */ 110 unsigned int strd_sz_n:4; /* Log 2 of stride size. */ 111 unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */ 112 unsigned int err_state:2; /* enum mlx5_rxq_err_state. */ 113 unsigned int strd_scatter_en:1; /* Scattered packets from a stride. */ 114 unsigned int lro:1; /* Enable LRO. */ 115 unsigned int dynf_meta:1; /* Dynamic metadata is configured. */ 116 volatile uint32_t *rq_db; 117 volatile uint32_t *cq_db; 118 uint16_t port_id; 119 uint32_t rq_ci; 120 uint16_t consumed_strd; /* Number of consumed strides in WQE. */ 121 uint32_t rq_pi; 122 uint32_t cq_ci; 123 uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */ 124 union { 125 struct rxq_zip zip; /* Compressed context. */ 126 uint16_t decompressed; 127 /* Number of ready mbufs decompressed from the CQ. */ 128 }; 129 struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */ 130 uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */ 131 volatile void *wqes; 132 volatile struct mlx5_cqe(*cqes)[]; 133 RTE_STD_C11 134 union { 135 struct rte_mbuf *(*elts)[]; 136 struct mlx5_mprq_buf *(*mprq_bufs)[]; 137 }; 138 struct rte_mempool *mp; 139 struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */ 140 struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */ 141 struct mlx5_dev_ctx_shared *sh; /* Shared context. */ 142 uint16_t idx; /* Queue index. */ 143 struct mlx5_rxq_stats stats; 144 rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx. */ 145 struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ 146 void *cq_uar; /* Verbs CQ user access region. */ 147 uint32_t cqn; /* CQ number. */ 148 uint8_t cq_arm_sn; /* CQ arm seq number. */ 149 #ifndef RTE_ARCH_64 150 rte_spinlock_t *uar_lock_cq; 151 /* CQ (UAR) access lock required for 32bit implementations */ 152 #endif 153 uint32_t tunnel; /* Tunnel information. */ 154 uint64_t flow_meta_mask; 155 int32_t flow_meta_offset; 156 } __rte_cache_aligned; 157 158 enum mlx5_rxq_type { 159 MLX5_RXQ_TYPE_STANDARD, /* Standard Rx queue. */ 160 MLX5_RXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */ 161 MLX5_RXQ_TYPE_UNDEFINED, 162 }; 163 164 /* RX queue control descriptor. */ 165 struct mlx5_rxq_ctrl { 166 struct mlx5_rxq_data rxq; /* Data path structure. */ 167 LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */ 168 rte_atomic32_t refcnt; /* Reference counter. */ 169 struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */ 170 struct mlx5_priv *priv; /* Back pointer to private data. */ 171 enum mlx5_rxq_type type; /* Rxq type. */ 172 unsigned int socket; /* CPU socket ID for allocations. */ 173 unsigned int irq:1; /* Whether IRQ is enabled. */ 174 uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */ 175 uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */ 176 uint32_t wqn; /* WQ number. */ 177 uint16_t dump_file_n; /* Number of dump files. */ 178 struct mlx5_devx_dbr_page *rq_dbrec_page; 179 uint64_t rq_dbr_offset; 180 /* Storing RQ door-bell information, needed when freeing door-bell. */ 181 struct mlx5_devx_dbr_page *cq_dbrec_page; 182 uint64_t cq_dbr_offset; 183 /* Storing CQ door-bell information, needed when freeing door-bell. */ 184 void *wq_umem; /* WQ buffer registration info. */ 185 void *cq_umem; /* CQ buffer registration info. */ 186 struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */ 187 }; 188 189 /* TX queue send local data. */ 190 __extension__ 191 struct mlx5_txq_local { 192 struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */ 193 struct rte_mbuf *mbuf; /* first mbuf to process. */ 194 uint16_t pkts_copy; /* packets copied to elts. */ 195 uint16_t pkts_sent; /* packets sent. */ 196 uint16_t pkts_loop; /* packets sent on loop entry. */ 197 uint16_t elts_free; /* available elts remain. */ 198 uint16_t wqe_free; /* available wqe remain. */ 199 uint16_t mbuf_off; /* data offset in current mbuf. */ 200 uint16_t mbuf_nseg; /* number of remaining mbuf. */ 201 }; 202 203 /* TX queue descriptor. */ 204 __extension__ 205 struct mlx5_txq_data { 206 uint16_t elts_head; /* Current counter in (*elts)[]. */ 207 uint16_t elts_tail; /* Counter of first element awaiting completion. */ 208 uint16_t elts_comp; /* elts index since last completion request. */ 209 uint16_t elts_s; /* Number of mbuf elements. */ 210 uint16_t elts_m; /* Mask for mbuf elements indices. */ 211 /* Fields related to elts mbuf storage. */ 212 uint16_t wqe_ci; /* Consumer index for work queue. */ 213 uint16_t wqe_pi; /* Producer index for work queue. */ 214 uint16_t wqe_s; /* Number of WQ elements. */ 215 uint16_t wqe_m; /* Mask Number for WQ elements. */ 216 uint16_t wqe_comp; /* WQE index since last completion request. */ 217 uint16_t wqe_thres; /* WQE threshold to request completion in CQ. */ 218 /* WQ related fields. */ 219 uint16_t cq_ci; /* Consumer index for completion queue. */ 220 uint16_t cq_pi; /* Production index for completion queue. */ 221 uint16_t cqe_s; /* Number of CQ elements. */ 222 uint16_t cqe_m; /* Mask for CQ indices. */ 223 /* CQ related fields. */ 224 uint16_t elts_n:4; /* elts[] length (in log2). */ 225 uint16_t cqe_n:4; /* Number of CQ elements (in log2). */ 226 uint16_t wqe_n:4; /* Number of WQ elements (in log2). */ 227 uint16_t tso_en:1; /* When set hardware TSO is enabled. */ 228 uint16_t tunnel_en:1; 229 /* When set TX offload for tunneled packets are supported. */ 230 uint16_t swp_en:1; /* Whether SW parser is enabled. */ 231 uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */ 232 uint16_t db_nc:1; /* Doorbell mapped to non-cached region. */ 233 uint16_t db_heu:1; /* Doorbell heuristic write barrier. */ 234 uint16_t inlen_send; /* Ordinary send data inline size. */ 235 uint16_t inlen_empw; /* eMPW max packet size to inline. */ 236 uint16_t inlen_mode; /* Minimal data length to inline. */ 237 uint32_t qp_num_8s; /* QP number shifted by 8. */ 238 uint64_t offloads; /* Offloads for Tx Queue. */ 239 struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */ 240 struct mlx5_wqe *wqes; /* Work queue. */ 241 struct mlx5_wqe *wqes_end; /* Work queue array limit. */ 242 #ifdef RTE_LIBRTE_MLX5_DEBUG 243 uint32_t *fcqs; /* Free completion queue (debug extended). */ 244 #else 245 uint16_t *fcqs; /* Free completion queue. */ 246 #endif 247 volatile struct mlx5_cqe *cqes; /* Completion queue. */ 248 volatile uint32_t *qp_db; /* Work queue doorbell. */ 249 volatile uint32_t *cq_db; /* Completion queue doorbell. */ 250 uint16_t port_id; /* Port ID of device. */ 251 uint16_t idx; /* Queue index. */ 252 uint64_t ts_mask; /* Timestamp flag dynamic mask. */ 253 int32_t ts_offset; /* Timestamp field dynamic offset. */ 254 struct mlx5_dev_ctx_shared *sh; /* Shared context. */ 255 struct mlx5_txq_stats stats; /* TX queue counters. */ 256 #ifndef RTE_ARCH_64 257 rte_spinlock_t *uar_lock; 258 /* UAR access lock required for 32bit implementations */ 259 #endif 260 struct rte_mbuf *elts[0]; 261 /* Storage for queued packets, must be the last field. */ 262 } __rte_cache_aligned; 263 264 enum mlx5_txq_obj_type { 265 MLX5_TXQ_OBJ_TYPE_IBV, /* mlx5_txq_obj with ibv_wq. */ 266 MLX5_TXQ_OBJ_TYPE_DEVX_SQ, /* mlx5_txq_obj with mlx5_devx_sq. */ 267 MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN, 268 /* mlx5_txq_obj with mlx5_devx_tq and hairpin support. */ 269 }; 270 271 enum mlx5_txq_type { 272 MLX5_TXQ_TYPE_STANDARD, /* Standard Tx queue. */ 273 MLX5_TXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */ 274 }; 275 276 /* Verbs/DevX Tx queue elements. */ 277 struct mlx5_txq_obj { 278 LIST_ENTRY(mlx5_txq_obj) next; /* Pointer to the next element. */ 279 rte_atomic32_t refcnt; /* Reference counter. */ 280 struct mlx5_txq_ctrl *txq_ctrl; /* Pointer to the control queue. */ 281 enum mlx5_txq_obj_type type; /* The txq object type. */ 282 RTE_STD_C11 283 union { 284 struct { 285 void *cq; /* Completion Queue. */ 286 void *qp; /* Queue Pair. */ 287 }; 288 struct { 289 struct mlx5_devx_obj *sq; 290 /* DevX object for Sx queue. */ 291 struct mlx5_devx_obj *tis; /* The TIS object. */ 292 }; 293 struct { 294 struct rte_eth_dev *dev; 295 struct mlx5_devx_obj *cq_devx; 296 void *cq_umem; 297 void *cq_buf; 298 int64_t cq_dbrec_offset; 299 struct mlx5_devx_dbr_page *cq_dbrec_page; 300 struct mlx5_devx_obj *sq_devx; 301 void *sq_umem; 302 void *sq_buf; 303 int64_t sq_dbrec_offset; 304 struct mlx5_devx_dbr_page *sq_dbrec_page; 305 }; 306 }; 307 }; 308 309 /* TX queue control descriptor. */ 310 struct mlx5_txq_ctrl { 311 LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */ 312 rte_atomic32_t refcnt; /* Reference counter. */ 313 unsigned int socket; /* CPU socket ID for allocations. */ 314 enum mlx5_txq_type type; /* The txq ctrl type. */ 315 unsigned int max_inline_data; /* Max inline data. */ 316 unsigned int max_tso_header; /* Max TSO header size. */ 317 struct mlx5_txq_obj *obj; /* Verbs/DevX queue object. */ 318 struct mlx5_priv *priv; /* Back pointer to private data. */ 319 off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ 320 void *bf_reg; /* BlueFlame register from Verbs. */ 321 uint16_t dump_file_n; /* Number of dump files. */ 322 struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */ 323 struct mlx5_txq_data txq; /* Data path structure. */ 324 /* Must be the last field in the structure, contains elts[]. */ 325 }; 326 327 #define MLX5_TX_BFREG(txq) \ 328 (MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx]) 329 330 /* mlx5_rxq.c */ 331 332 extern uint8_t rss_hash_default_key[]; 333 334 int mlx5_check_mprq_support(struct rte_eth_dev *dev); 335 int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq); 336 int mlx5_mprq_enabled(struct rte_eth_dev *dev); 337 unsigned int mlx5_rxq_cqe_num(struct mlx5_rxq_data *rxq_data); 338 int mlx5_mprq_free_mp(struct rte_eth_dev *dev); 339 int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev); 340 int mlx5_rx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id); 341 int mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id); 342 int mlx5_rx_queue_start_primary(struct rte_eth_dev *dev, uint16_t queue_id); 343 int mlx5_rx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t queue_id); 344 int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 345 unsigned int socket, const struct rte_eth_rxconf *conf, 346 struct rte_mempool *mp); 347 int mlx5_rx_hairpin_queue_setup 348 (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 349 const struct rte_eth_hairpin_conf *hairpin_conf); 350 void mlx5_rx_queue_release(void *dpdk_rxq); 351 int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev); 352 void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); 353 int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); 354 int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); 355 int mlx5_rxq_obj_verify(struct rte_eth_dev *dev); 356 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, 357 uint16_t desc, unsigned int socket, 358 const struct rte_eth_rxconf *conf, 359 struct rte_mempool *mp); 360 struct mlx5_rxq_ctrl *mlx5_rxq_hairpin_new 361 (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 362 const struct rte_eth_hairpin_conf *hairpin_conf); 363 struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); 364 int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx); 365 int mlx5_rxq_verify(struct rte_eth_dev *dev); 366 int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); 367 int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev); 368 struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev, 369 const uint16_t *queues, 370 uint32_t queues_n); 371 int mlx5_ind_table_obj_release(struct rte_eth_dev *dev, 372 struct mlx5_ind_table_obj *ind_tbl); 373 uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev, 374 const uint8_t *rss_key, uint32_t rss_key_len, 375 uint64_t hash_fields, 376 const uint16_t *queues, uint32_t queues_n, 377 int tunnel __rte_unused); 378 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev, 379 const uint8_t *rss_key, uint32_t rss_key_len, 380 uint64_t hash_fields, 381 const uint16_t *queues, uint32_t queues_n); 382 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx); 383 int mlx5_hrxq_verify(struct rte_eth_dev *dev); 384 enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx); 385 struct mlx5_hrxq *mlx5_drop_action_create(struct rte_eth_dev *dev); 386 void mlx5_drop_action_destroy(struct rte_eth_dev *dev); 387 uint64_t mlx5_get_rx_port_offloads(void); 388 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev); 389 void mlx5_rxq_timestamp_set(struct rte_eth_dev *dev); 390 391 392 /* mlx5_txq.c */ 393 394 int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id); 395 int mlx5_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id); 396 int mlx5_tx_queue_start_primary(struct rte_eth_dev *dev, uint16_t queue_id); 397 int mlx5_tx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t queue_id); 398 int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 399 unsigned int socket, const struct rte_eth_txconf *conf); 400 int mlx5_tx_hairpin_queue_setup 401 (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 402 const struct rte_eth_hairpin_conf *hairpin_conf); 403 void mlx5_tx_queue_release(void *dpdk_txq); 404 int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd); 405 void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev); 406 struct mlx5_txq_obj *mlx5_txq_obj_new(struct rte_eth_dev *dev, uint16_t idx, 407 enum mlx5_txq_obj_type type); 408 struct mlx5_txq_obj *mlx5_txq_obj_get(struct rte_eth_dev *dev, uint16_t idx); 409 int mlx5_txq_obj_release(struct mlx5_txq_obj *txq_ibv); 410 int mlx5_txq_obj_verify(struct rte_eth_dev *dev); 411 struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, 412 uint16_t desc, unsigned int socket, 413 const struct rte_eth_txconf *conf); 414 struct mlx5_txq_ctrl *mlx5_txq_hairpin_new 415 (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 416 const struct rte_eth_hairpin_conf *hairpin_conf); 417 struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx); 418 int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); 419 int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); 420 int mlx5_txq_verify(struct rte_eth_dev *dev); 421 void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl); 422 void txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl); 423 uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev); 424 void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev); 425 426 /* mlx5_rxtx.c */ 427 428 extern uint32_t mlx5_ptype_table[]; 429 extern uint8_t mlx5_cksum_table[]; 430 extern uint8_t mlx5_swp_types_table[]; 431 432 void mlx5_set_ptype_table(void); 433 void mlx5_set_cksum_table(void); 434 void mlx5_set_swp_types_table(void); 435 uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); 436 void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq); 437 __rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec); 438 void mlx5_mprq_buf_free_cb(void *addr, void *opaque); 439 void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf); 440 uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, 441 uint16_t pkts_n); 442 uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, 443 uint16_t pkts_n); 444 uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, 445 uint16_t pkts_n); 446 int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); 447 int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset); 448 uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id); 449 void mlx5_dump_debug_information(const char *path, const char *title, 450 const void *buf, unsigned int len); 451 int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev, 452 const struct mlx5_mp_arg_queue_state_modify *sm); 453 void mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 454 struct rte_eth_rxq_info *qinfo); 455 void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 456 struct rte_eth_txq_info *qinfo); 457 int mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, 458 struct rte_eth_burst_mode *mode); 459 int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, 460 struct rte_eth_burst_mode *mode); 461 462 /* Vectorized version of mlx5_rxtx.c */ 463 int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data); 464 int mlx5_check_vec_rx_support(struct rte_eth_dev *dev); 465 uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, 466 uint16_t pkts_n); 467 468 /* mlx5_mr.c */ 469 470 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl); 471 uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr); 472 uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb); 473 uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr, 474 struct rte_mempool *mp); 475 int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova, 476 size_t len); 477 int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova, 478 size_t len); 479 480 /** 481 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and 482 * 64bit architectures. 483 * 484 * @param val 485 * value to write in CPU endian format. 486 * @param addr 487 * Address to write to. 488 * @param lock 489 * Address of the lock to use for that UAR access. 490 */ 491 static __rte_always_inline void 492 __mlx5_uar_write64_relaxed(uint64_t val, void *addr, 493 rte_spinlock_t *lock __rte_unused) 494 { 495 #ifdef RTE_ARCH_64 496 *(uint64_t *)addr = val; 497 #else /* !RTE_ARCH_64 */ 498 rte_spinlock_lock(lock); 499 *(uint32_t *)addr = val; 500 rte_io_wmb(); 501 *((uint32_t *)addr + 1) = val >> 32; 502 rte_spinlock_unlock(lock); 503 #endif 504 } 505 506 /** 507 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and 508 * 64bit architectures while guaranteeing the order of execution with the 509 * code being executed. 510 * 511 * @param val 512 * value to write in CPU endian format. 513 * @param addr 514 * Address to write to. 515 * @param lock 516 * Address of the lock to use for that UAR access. 517 */ 518 static __rte_always_inline void 519 __mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock) 520 { 521 rte_io_wmb(); 522 __mlx5_uar_write64_relaxed(val, addr, lock); 523 } 524 525 /* Assist macros, used instead of directly calling the functions they wrap. */ 526 #ifdef RTE_ARCH_64 527 #define mlx5_uar_write64_relaxed(val, dst, lock) \ 528 __mlx5_uar_write64_relaxed(val, dst, NULL) 529 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL) 530 #else 531 #define mlx5_uar_write64_relaxed(val, dst, lock) \ 532 __mlx5_uar_write64_relaxed(val, dst, lock) 533 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock) 534 #endif 535 536 /** 537 * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the 538 * cloned mbuf is allocated is returned instead. 539 * 540 * @param buf 541 * Pointer to mbuf. 542 * 543 * @return 544 * Memory pool where data is located for given mbuf. 545 */ 546 static inline struct rte_mempool * 547 mlx5_mb2mp(struct rte_mbuf *buf) 548 { 549 if (unlikely(RTE_MBUF_CLONED(buf))) 550 return rte_mbuf_from_indirect(buf)->pool; 551 return buf->pool; 552 } 553 554 /** 555 * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx 556 * as mempool is pre-configured and static. 557 * 558 * @param rxq 559 * Pointer to Rx queue structure. 560 * @param addr 561 * Address to search. 562 * 563 * @return 564 * Searched LKey on success, UINT32_MAX on no match. 565 */ 566 static __rte_always_inline uint32_t 567 mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) 568 { 569 struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; 570 uint32_t lkey; 571 572 /* Linear search on MR cache array. */ 573 lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru, 574 MLX5_MR_CACHE_N, addr); 575 if (likely(lkey != UINT32_MAX)) 576 return lkey; 577 /* Take slower bottom-half (Binary Search) on miss. */ 578 return mlx5_rx_addr2mr_bh(rxq, addr); 579 } 580 581 #define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr)) 582 583 /** 584 * Query LKey from a packet buffer for Tx. If not found, add the mempool. 585 * 586 * @param txq 587 * Pointer to Tx queue structure. 588 * @param addr 589 * Address to search. 590 * 591 * @return 592 * Searched LKey on success, UINT32_MAX on no match. 593 */ 594 static __rte_always_inline uint32_t 595 mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) 596 { 597 struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl; 598 uintptr_t addr = (uintptr_t)mb->buf_addr; 599 uint32_t lkey; 600 601 /* Check generation bit to see if there's any change on existing MRs. */ 602 if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen)) 603 mlx5_mr_flush_local_cache(mr_ctrl); 604 /* Linear search on MR cache array. */ 605 lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru, 606 MLX5_MR_CACHE_N, addr); 607 if (likely(lkey != UINT32_MAX)) 608 return lkey; 609 /* Take slower bottom-half on miss. */ 610 return mlx5_tx_mb2mr_bh(txq, mb); 611 } 612 613 /** 614 * Ring TX queue doorbell and flush the update if requested. 615 * 616 * @param txq 617 * Pointer to TX queue structure. 618 * @param wqe 619 * Pointer to the last WQE posted in the NIC. 620 * @param cond 621 * Request for write memory barrier after BlueFlame update. 622 */ 623 static __rte_always_inline void 624 mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe, 625 int cond) 626 { 627 uint64_t *dst = MLX5_TX_BFREG(txq); 628 volatile uint64_t *src = ((volatile uint64_t *)wqe); 629 630 rte_cio_wmb(); 631 *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci); 632 /* Ensure ordering between DB record and BF copy. */ 633 rte_wmb(); 634 mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock); 635 if (cond) 636 rte_wmb(); 637 } 638 639 /** 640 * Ring TX queue doorbell and flush the update by write memory barrier. 641 * 642 * @param txq 643 * Pointer to TX queue structure. 644 * @param wqe 645 * Pointer to the last WQE posted in the NIC. 646 */ 647 static __rte_always_inline void 648 mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe) 649 { 650 mlx5_tx_dbrec_cond_wmb(txq, wqe, 1); 651 } 652 653 /** 654 * Convert timestamp from HW format to linear counter 655 * from Packet Pacing Clock Queue CQE timestamp format. 656 * 657 * @param sh 658 * Pointer to the device shared context. Might be needed 659 * to convert according current device configuration. 660 * @param ts 661 * Timestamp from CQE to convert. 662 * @return 663 * UTC in nanoseconds 664 */ 665 static __rte_always_inline uint64_t 666 mlx5_txpp_convert_rx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t ts) 667 { 668 RTE_SET_USED(sh); 669 return (ts & UINT32_MAX) + (ts >> 32) * NS_PER_S; 670 } 671 672 /** 673 * Convert timestamp from mbuf format to linear counter 674 * of Clock Queue completions (24 bits) 675 * 676 * @param sh 677 * Pointer to the device shared context to fetch Tx 678 * packet pacing timestamp and parameters. 679 * @param ts 680 * Timestamp from mbuf to convert. 681 * @return 682 * positive or zero value - completion ID to wait 683 * negative value - conversion error 684 */ 685 static __rte_always_inline int32_t 686 mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts) 687 { 688 uint64_t ts, ci; 689 uint32_t tick; 690 691 do { 692 /* 693 * Read atomically two uint64_t fields and compare lsb bits. 694 * It there is no match - the timestamp was updated in 695 * the service thread, data should be re-read. 696 */ 697 rte_compiler_barrier(); 698 ci = rte_atomic64_read(&sh->txpp.ts.ci_ts); 699 ts = rte_atomic64_read(&sh->txpp.ts.ts); 700 rte_compiler_barrier(); 701 if (!((ts ^ ci) << (64 - MLX5_CQ_INDEX_WIDTH))) 702 break; 703 } while (true); 704 /* Perform the skew correction, positive value to send earlier. */ 705 mts -= sh->txpp.skew; 706 mts -= ts; 707 if (unlikely(mts >= UINT64_MAX / 2)) { 708 /* We have negative integer, mts is in the past. */ 709 rte_atomic32_inc(&sh->txpp.err_ts_past); 710 return -1; 711 } 712 tick = sh->txpp.tick; 713 MLX5_ASSERT(tick); 714 /* Convert delta to completions, round up. */ 715 mts = (mts + tick - 1) / tick; 716 if (unlikely(mts >= (1 << MLX5_CQ_INDEX_WIDTH) / 2 - 1)) { 717 /* We have mts is too distant future. */ 718 rte_atomic32_inc(&sh->txpp.err_ts_future); 719 return -1; 720 } 721 mts <<= 64 - MLX5_CQ_INDEX_WIDTH; 722 ci += mts; 723 ci >>= 64 - MLX5_CQ_INDEX_WIDTH; 724 return ci; 725 } 726 727 #endif /* RTE_PMD_MLX5_RXTX_H_ */ 728