1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #ifndef RTE_PMD_MLX5_RXTX_H_ 7 #define RTE_PMD_MLX5_RXTX_H_ 8 9 #include <stddef.h> 10 #include <stdint.h> 11 #include <sys/queue.h> 12 13 /* Verbs header. */ 14 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 15 #ifdef PEDANTIC 16 #pragma GCC diagnostic ignored "-Wpedantic" 17 #endif 18 #include <infiniband/verbs.h> 19 #include <infiniband/mlx5dv.h> 20 #ifdef PEDANTIC 21 #pragma GCC diagnostic error "-Wpedantic" 22 #endif 23 24 #include <rte_mbuf.h> 25 #include <rte_mempool.h> 26 #include <rte_common.h> 27 #include <rte_hexdump.h> 28 #include <rte_atomic.h> 29 #include <rte_spinlock.h> 30 #include <rte_io.h> 31 #include <rte_bus_pci.h> 32 #include <rte_malloc.h> 33 34 #include "mlx5_utils.h" 35 #include "mlx5.h" 36 #include "mlx5_mr.h" 37 #include "mlx5_autoconf.h" 38 #include "mlx5_defs.h" 39 #include "mlx5_prm.h" 40 #include "mlx5_glue.h" 41 42 /* Support tunnel matching. */ 43 #define MLX5_FLOW_TUNNEL 8 44 45 struct mlx5_rxq_stats { 46 #ifdef MLX5_PMD_SOFT_COUNTERS 47 uint64_t ipackets; /**< Total of successfully received packets. */ 48 uint64_t ibytes; /**< Total of successfully received bytes. */ 49 #endif 50 uint64_t idropped; /**< Total of packets dropped when RX ring full. */ 51 uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */ 52 }; 53 54 struct mlx5_txq_stats { 55 #ifdef MLX5_PMD_SOFT_COUNTERS 56 uint64_t opackets; /**< Total of successfully sent packets. */ 57 uint64_t obytes; /**< Total of successfully sent bytes. */ 58 #endif 59 uint64_t oerrors; /**< Total number of failed transmitted packets. */ 60 }; 61 62 struct mlx5_priv; 63 64 /* Compressed CQE context. */ 65 struct rxq_zip { 66 uint16_t ai; /* Array index. */ 67 uint16_t ca; /* Current array index. */ 68 uint16_t na; /* Next array index. */ 69 uint16_t cq_ci; /* The next CQE. */ 70 uint32_t cqe_cnt; /* Number of CQEs. */ 71 }; 72 73 /* Multi-Packet RQ buffer header. */ 74 struct mlx5_mprq_buf { 75 struct rte_mempool *mp; 76 rte_atomic16_t refcnt; /* Atomically accessed refcnt. */ 77 uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */ 78 struct rte_mbuf_ext_shared_info shinfos[]; 79 /* 80 * Shared information per stride. 81 * More memory will be allocated for the first stride head-room and for 82 * the strides data. 83 */ 84 } __rte_cache_aligned; 85 86 /* Get pointer to the first stride. */ 87 #define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \ 88 sizeof(struct mlx5_mprq_buf) + \ 89 (strd_n) * \ 90 sizeof(struct rte_mbuf_ext_shared_info) + \ 91 RTE_PKTMBUF_HEADROOM)) 92 93 #define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6 94 #define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9 95 96 enum mlx5_rxq_err_state { 97 MLX5_RXQ_ERR_STATE_NO_ERROR = 0, 98 MLX5_RXQ_ERR_STATE_NEED_RESET, 99 MLX5_RXQ_ERR_STATE_NEED_READY, 100 }; 101 102 /* RX queue descriptor. */ 103 struct mlx5_rxq_data { 104 unsigned int csum:1; /* Enable checksum offloading. */ 105 unsigned int hw_timestamp:1; /* Enable HW timestamp. */ 106 unsigned int vlan_strip:1; /* Enable VLAN stripping. */ 107 unsigned int crc_present:1; /* CRC must be subtracted. */ 108 unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */ 109 unsigned int cqe_n:4; /* Log 2 of CQ elements. */ 110 unsigned int elts_n:4; /* Log 2 of Mbufs. */ 111 unsigned int rss_hash:1; /* RSS hash result is enabled. */ 112 unsigned int mark:1; /* Marked flow available on the queue. */ 113 unsigned int strd_num_n:5; /* Log 2 of the number of stride. */ 114 unsigned int strd_sz_n:4; /* Log 2 of stride size. */ 115 unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */ 116 unsigned int err_state:2; /* enum mlx5_rxq_err_state. */ 117 unsigned int strd_headroom_en:1; /* Enable mbuf headroom in MPRQ. */ 118 unsigned int lro:1; /* Enable LRO. */ 119 unsigned int :1; /* Remaining bits. */ 120 volatile uint32_t *rq_db; 121 volatile uint32_t *cq_db; 122 uint16_t port_id; 123 uint32_t rq_ci; 124 uint16_t consumed_strd; /* Number of consumed strides in WQE. */ 125 uint32_t rq_pi; 126 uint32_t cq_ci; 127 uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */ 128 union { 129 struct rxq_zip zip; /* Compressed context. */ 130 uint16_t decompressed; 131 /* Number of ready mbufs decompressed from the CQ. */ 132 }; 133 struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */ 134 uint16_t mprq_max_memcpy_len; /* Maximum size of packet to memcpy. */ 135 volatile void *wqes; 136 volatile struct mlx5_cqe(*cqes)[]; 137 RTE_STD_C11 138 union { 139 struct rte_mbuf *(*elts)[]; 140 struct mlx5_mprq_buf *(*mprq_bufs)[]; 141 }; 142 struct rte_mempool *mp; 143 struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */ 144 struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */ 145 uint16_t idx; /* Queue index. */ 146 struct mlx5_rxq_stats stats; 147 uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */ 148 struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ 149 void *cq_uar; /* CQ user access region. */ 150 uint32_t cqn; /* CQ number. */ 151 uint8_t cq_arm_sn; /* CQ arm seq number. */ 152 #ifndef RTE_ARCH_64 153 rte_spinlock_t *uar_lock_cq; 154 /* CQ (UAR) access lock required for 32bit implementations */ 155 #endif 156 uint32_t tunnel; /* Tunnel information. */ 157 } __rte_cache_aligned; 158 159 enum mlx5_rxq_obj_type { 160 MLX5_RXQ_OBJ_TYPE_IBV, /* mlx5_rxq_obj with ibv_wq. */ 161 MLX5_RXQ_OBJ_TYPE_DEVX_RQ, /* mlx5_rxq_obj with mlx5_devx_rq. */ 162 }; 163 164 /* Verbs/DevX Rx queue elements. */ 165 struct mlx5_rxq_obj { 166 LIST_ENTRY(mlx5_rxq_obj) next; /* Pointer to the next element. */ 167 rte_atomic32_t refcnt; /* Reference counter. */ 168 struct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */ 169 struct ibv_cq *cq; /* Completion Queue. */ 170 enum mlx5_rxq_obj_type type; 171 RTE_STD_C11 172 union { 173 struct ibv_wq *wq; /* Work Queue. */ 174 struct mlx5_devx_obj *rq; /* DevX object for Rx Queue. */ 175 }; 176 struct ibv_comp_channel *channel; 177 }; 178 179 /* RX queue control descriptor. */ 180 struct mlx5_rxq_ctrl { 181 struct mlx5_rxq_data rxq; /* Data path structure. */ 182 LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */ 183 rte_atomic32_t refcnt; /* Reference counter. */ 184 struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */ 185 struct mlx5_priv *priv; /* Back pointer to private data. */ 186 unsigned int socket; /* CPU socket ID for allocations. */ 187 unsigned int irq:1; /* Whether IRQ is enabled. */ 188 unsigned int dbr_umem_id_valid:1; /* dbr_umem_id holds a valid value. */ 189 uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */ 190 uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */ 191 uint32_t wqn; /* WQ number. */ 192 uint16_t dump_file_n; /* Number of dump files. */ 193 uint32_t dbr_umem_id; /* Storing door-bell information, */ 194 uint64_t dbr_offset; /* needed when freeing door-bell. */ 195 struct mlx5dv_devx_umem *wq_umem; /* WQ buffer registration info. */ 196 }; 197 198 enum mlx5_ind_tbl_type { 199 MLX5_IND_TBL_TYPE_IBV, 200 MLX5_IND_TBL_TYPE_DEVX, 201 }; 202 203 /* Indirection table. */ 204 struct mlx5_ind_table_obj { 205 LIST_ENTRY(mlx5_ind_table_obj) next; /* Pointer to the next element. */ 206 rte_atomic32_t refcnt; /* Reference counter. */ 207 enum mlx5_ind_tbl_type type; 208 RTE_STD_C11 209 union { 210 struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */ 211 struct mlx5_devx_obj *rqt; /* DevX RQT object. */ 212 }; 213 uint32_t queues_n; /**< Number of queues in the list. */ 214 uint16_t queues[]; /**< Queue list. */ 215 }; 216 217 /* Hash Rx queue. */ 218 struct mlx5_hrxq { 219 LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */ 220 rte_atomic32_t refcnt; /* Reference counter. */ 221 struct mlx5_ind_table_obj *ind_table; /* Indirection table. */ 222 RTE_STD_C11 223 union { 224 struct ibv_qp *qp; /* Verbs queue pair. */ 225 struct mlx5_devx_obj *tir; /* DevX TIR object. */ 226 }; 227 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 228 void *action; /* DV QP action pointer. */ 229 #endif 230 uint64_t hash_fields; /* Verbs Hash fields. */ 231 uint32_t rss_key_len; /* Hash key length in bytes. */ 232 uint8_t rss_key[]; /* Hash key. */ 233 }; 234 235 /* TX queue send local data. */ 236 __extension__ 237 struct mlx5_txq_local { 238 struct mlx5_wqe *wqe_last; /* last sent WQE pointer. */ 239 struct rte_mbuf *mbuf; /* first mbuf to process. */ 240 uint16_t pkts_copy; /* packets copied to elts. */ 241 uint16_t pkts_sent; /* packets sent. */ 242 uint16_t pkts_loop; /* packets sent on loop entry. */ 243 uint16_t elts_free; /* available elts remain. */ 244 uint16_t wqe_free; /* available wqe remain. */ 245 uint16_t mbuf_off; /* data offset in current mbuf. */ 246 uint16_t mbuf_nseg; /* number of remaining mbuf. */ 247 }; 248 249 /* TX queue descriptor. */ 250 __extension__ 251 struct mlx5_txq_data { 252 uint16_t elts_head; /* Current counter in (*elts)[]. */ 253 uint16_t elts_tail; /* Counter of first element awaiting completion. */ 254 uint16_t elts_comp; /* elts index since last completion request. */ 255 uint16_t elts_s; /* Number of mbuf elements. */ 256 uint16_t elts_m; /* Mask for mbuf elements indices. */ 257 /* Fields related to elts mbuf storage. */ 258 uint16_t wqe_ci; /* Consumer index for work queue. */ 259 uint16_t wqe_pi; /* Producer index for work queue. */ 260 uint16_t wqe_s; /* Number of WQ elements. */ 261 uint16_t wqe_m; /* Mask Number for WQ elements. */ 262 uint16_t wqe_comp; /* WQE index since last completion request. */ 263 uint16_t wqe_thres; /* WQE threshold to request completion in CQ. */ 264 /* WQ related fields. */ 265 uint16_t cq_ci; /* Consumer index for completion queue. */ 266 #ifndef NDEBUG 267 uint16_t cq_pi; /* Counter of issued CQE "always" requests. */ 268 #endif 269 uint16_t cqe_s; /* Number of CQ elements. */ 270 uint16_t cqe_m; /* Mask for CQ indices. */ 271 /* CQ related fields. */ 272 uint16_t elts_n:4; /* elts[] length (in log2). */ 273 uint16_t cqe_n:4; /* Number of CQ elements (in log2). */ 274 uint16_t wqe_n:4; /* Number of WQ elements (in log2). */ 275 uint16_t tso_en:1; /* When set hardware TSO is enabled. */ 276 uint16_t tunnel_en:1; 277 /* When set TX offload for tunneled packets are supported. */ 278 uint16_t swp_en:1; /* Whether SW parser is enabled. */ 279 uint16_t vlan_en:1; /* VLAN insertion in WQE is supported. */ 280 uint16_t inlen_send; /* Ordinary send data inline size. */ 281 uint16_t inlen_empw; /* eMPW max packet size to inline. */ 282 uint16_t inlen_mode; /* Minimal data length to inline. */ 283 uint32_t qp_num_8s; /* QP number shifted by 8. */ 284 uint64_t offloads; /* Offloads for Tx Queue. */ 285 struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */ 286 struct mlx5_wqe *wqes; /* Work queue. */ 287 struct mlx5_wqe *wqes_end; /* Work queue array limit. */ 288 volatile struct mlx5_cqe *cqes; /* Completion queue. */ 289 volatile uint32_t *qp_db; /* Work queue doorbell. */ 290 volatile uint32_t *cq_db; /* Completion queue doorbell. */ 291 uint16_t port_id; /* Port ID of device. */ 292 uint16_t idx; /* Queue index. */ 293 struct mlx5_txq_stats stats; /* TX queue counters. */ 294 #ifndef RTE_ARCH_64 295 rte_spinlock_t *uar_lock; 296 /* UAR access lock required for 32bit implementations */ 297 #endif 298 struct rte_mbuf *elts[0]; 299 /* Storage for queued packets, must be the last field. */ 300 } __rte_cache_aligned; 301 302 /* Verbs Rx queue elements. */ 303 struct mlx5_txq_ibv { 304 LIST_ENTRY(mlx5_txq_ibv) next; /* Pointer to the next element. */ 305 rte_atomic32_t refcnt; /* Reference counter. */ 306 struct mlx5_txq_ctrl *txq_ctrl; /* Pointer to the control queue. */ 307 struct ibv_cq *cq; /* Completion Queue. */ 308 struct ibv_qp *qp; /* Queue Pair. */ 309 }; 310 311 /* TX queue control descriptor. */ 312 struct mlx5_txq_ctrl { 313 LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */ 314 rte_atomic32_t refcnt; /* Reference counter. */ 315 unsigned int socket; /* CPU socket ID for allocations. */ 316 unsigned int max_inline_data; /* Max inline data. */ 317 unsigned int max_tso_header; /* Max TSO header size. */ 318 struct mlx5_txq_ibv *ibv; /* Verbs queue object. */ 319 struct mlx5_priv *priv; /* Back pointer to private data. */ 320 off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ 321 void *bf_reg; /* BlueFlame register from Verbs. */ 322 uint16_t dump_file_n; /* Number of dump files. */ 323 struct mlx5_txq_data txq; /* Data path structure. */ 324 /* Must be the last field in the structure, contains elts[]. */ 325 }; 326 327 #define MLX5_TX_BFREG(txq) \ 328 (MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx]) 329 330 /* mlx5_rxq.c */ 331 332 extern uint8_t rss_hash_default_key[]; 333 334 int mlx5_check_mprq_support(struct rte_eth_dev *dev); 335 int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq); 336 int mlx5_mprq_enabled(struct rte_eth_dev *dev); 337 int mlx5_mprq_free_mp(struct rte_eth_dev *dev); 338 int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev); 339 int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 340 unsigned int socket, const struct rte_eth_rxconf *conf, 341 struct rte_mempool *mp); 342 void mlx5_rx_queue_release(void *dpdk_rxq); 343 int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev); 344 void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); 345 int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); 346 int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); 347 struct mlx5_rxq_obj *mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx, 348 enum mlx5_rxq_obj_type type); 349 int mlx5_rxq_obj_verify(struct rte_eth_dev *dev); 350 struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, 351 uint16_t desc, unsigned int socket, 352 const struct rte_eth_rxconf *conf, 353 struct rte_mempool *mp); 354 struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); 355 int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx); 356 int mlx5_rxq_verify(struct rte_eth_dev *dev); 357 int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); 358 int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev); 359 struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, 360 const uint8_t *rss_key, uint32_t rss_key_len, 361 uint64_t hash_fields, 362 const uint16_t *queues, uint32_t queues_n, 363 int tunnel __rte_unused); 364 struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, 365 const uint8_t *rss_key, uint32_t rss_key_len, 366 uint64_t hash_fields, 367 const uint16_t *queues, uint32_t queues_n); 368 int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq); 369 int mlx5_hrxq_verify(struct rte_eth_dev *dev); 370 struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev); 371 void mlx5_hrxq_drop_release(struct rte_eth_dev *dev); 372 uint64_t mlx5_get_rx_port_offloads(void); 373 uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev); 374 375 /* mlx5_txq.c */ 376 377 int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 378 unsigned int socket, const struct rte_eth_txconf *conf); 379 void mlx5_tx_queue_release(void *dpdk_txq); 380 int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd); 381 struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); 382 struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); 383 int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv); 384 int mlx5_txq_ibv_verify(struct rte_eth_dev *dev); 385 struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, 386 uint16_t desc, unsigned int socket, 387 const struct rte_eth_txconf *conf); 388 struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx); 389 int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); 390 int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); 391 int mlx5_txq_verify(struct rte_eth_dev *dev); 392 void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl); 393 uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev); 394 395 /* mlx5_rxtx.c */ 396 397 extern uint32_t mlx5_ptype_table[]; 398 extern uint8_t mlx5_cksum_table[]; 399 extern uint8_t mlx5_swp_types_table[]; 400 401 void mlx5_set_ptype_table(void); 402 void mlx5_set_cksum_table(void); 403 void mlx5_set_swp_types_table(void); 404 __rte_noinline int mlx5_tx_error_cqe_handle 405 (struct mlx5_txq_data *restrict txq, 406 volatile struct mlx5_err_cqe *err_cqe); 407 uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); 408 void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq); 409 __rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, 410 uint8_t mbuf_prepare); 411 void mlx5_mprq_buf_free_cb(void *addr, void *opaque); 412 void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf); 413 uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, 414 uint16_t pkts_n); 415 uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, 416 uint16_t pkts_n); 417 uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, 418 uint16_t pkts_n); 419 int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); 420 int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset); 421 uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id); 422 void mlx5_dump_debug_information(const char *path, const char *title, 423 const void *buf, unsigned int len); 424 int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev, 425 const struct mlx5_mp_arg_queue_state_modify *sm); 426 427 /* Vectorized version of mlx5_rxtx.c */ 428 int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data); 429 int mlx5_check_vec_rx_support(struct rte_eth_dev *dev); 430 uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, 431 uint16_t pkts_n); 432 433 /* mlx5_mr.c */ 434 435 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl); 436 uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr); 437 uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb); 438 uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr, 439 struct rte_mempool *mp); 440 int mlx5_dma_map(struct rte_pci_device *pdev, void *addr, uint64_t iova, 441 size_t len); 442 int mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr, uint64_t iova, 443 size_t len); 444 445 /** 446 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and 447 * 64bit architectures. 448 * 449 * @param val 450 * value to write in CPU endian format. 451 * @param addr 452 * Address to write to. 453 * @param lock 454 * Address of the lock to use for that UAR access. 455 */ 456 static __rte_always_inline void 457 __mlx5_uar_write64_relaxed(uint64_t val, void *addr, 458 rte_spinlock_t *lock __rte_unused) 459 { 460 #ifdef RTE_ARCH_64 461 *(uint64_t *)addr = val; 462 #else /* !RTE_ARCH_64 */ 463 rte_spinlock_lock(lock); 464 *(uint32_t *)addr = val; 465 rte_io_wmb(); 466 *((uint32_t *)addr + 1) = val >> 32; 467 rte_spinlock_unlock(lock); 468 #endif 469 } 470 471 /** 472 * Provide safe 64bit store operation to mlx5 UAR region for both 32bit and 473 * 64bit architectures while guaranteeing the order of execution with the 474 * code being executed. 475 * 476 * @param val 477 * value to write in CPU endian format. 478 * @param addr 479 * Address to write to. 480 * @param lock 481 * Address of the lock to use for that UAR access. 482 */ 483 static __rte_always_inline void 484 __mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock) 485 { 486 rte_io_wmb(); 487 __mlx5_uar_write64_relaxed(val, addr, lock); 488 } 489 490 /* Assist macros, used instead of directly calling the functions they wrap. */ 491 #ifdef RTE_ARCH_64 492 #define mlx5_uar_write64_relaxed(val, dst, lock) \ 493 __mlx5_uar_write64_relaxed(val, dst, NULL) 494 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, NULL) 495 #else 496 #define mlx5_uar_write64_relaxed(val, dst, lock) \ 497 __mlx5_uar_write64_relaxed(val, dst, lock) 498 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock) 499 #endif 500 501 /* CQE status. */ 502 enum mlx5_cqe_status { 503 MLX5_CQE_STATUS_SW_OWN = -1, 504 MLX5_CQE_STATUS_HW_OWN = -2, 505 MLX5_CQE_STATUS_ERR = -3, 506 }; 507 508 /** 509 * Check whether CQE is valid. 510 * 511 * @param cqe 512 * Pointer to CQE. 513 * @param cqes_n 514 * Size of completion queue. 515 * @param ci 516 * Consumer index. 517 * 518 * @return 519 * The CQE status. 520 */ 521 static __rte_always_inline enum mlx5_cqe_status 522 check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n, 523 const uint16_t ci) 524 { 525 const uint16_t idx = ci & cqes_n; 526 const uint8_t op_own = cqe->op_own; 527 const uint8_t op_owner = MLX5_CQE_OWNER(op_own); 528 const uint8_t op_code = MLX5_CQE_OPCODE(op_own); 529 530 if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID))) 531 return MLX5_CQE_STATUS_HW_OWN; 532 rte_cio_rmb(); 533 if (unlikely(op_code == MLX5_CQE_RESP_ERR || 534 op_code == MLX5_CQE_REQ_ERR)) 535 return MLX5_CQE_STATUS_ERR; 536 return MLX5_CQE_STATUS_SW_OWN; 537 } 538 539 /** 540 * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the 541 * cloned mbuf is allocated is returned instead. 542 * 543 * @param buf 544 * Pointer to mbuf. 545 * 546 * @return 547 * Memory pool where data is located for given mbuf. 548 */ 549 static inline struct rte_mempool * 550 mlx5_mb2mp(struct rte_mbuf *buf) 551 { 552 if (unlikely(RTE_MBUF_CLONED(buf))) 553 return rte_mbuf_from_indirect(buf)->pool; 554 return buf->pool; 555 } 556 557 /** 558 * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx 559 * as mempool is pre-configured and static. 560 * 561 * @param rxq 562 * Pointer to Rx queue structure. 563 * @param addr 564 * Address to search. 565 * 566 * @return 567 * Searched LKey on success, UINT32_MAX on no match. 568 */ 569 static __rte_always_inline uint32_t 570 mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) 571 { 572 struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; 573 uint32_t lkey; 574 575 /* Linear search on MR cache array. */ 576 lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru, 577 MLX5_MR_CACHE_N, addr); 578 if (likely(lkey != UINT32_MAX)) 579 return lkey; 580 /* Take slower bottom-half (Binary Search) on miss. */ 581 return mlx5_rx_addr2mr_bh(rxq, addr); 582 } 583 584 #define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr)) 585 586 /** 587 * Query LKey from a packet buffer for Tx. If not found, add the mempool. 588 * 589 * @param txq 590 * Pointer to Tx queue structure. 591 * @param addr 592 * Address to search. 593 * 594 * @return 595 * Searched LKey on success, UINT32_MAX on no match. 596 */ 597 static __rte_always_inline uint32_t 598 mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) 599 { 600 struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl; 601 uintptr_t addr = (uintptr_t)mb->buf_addr; 602 uint32_t lkey; 603 604 /* Check generation bit to see if there's any change on existing MRs. */ 605 if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen)) 606 mlx5_mr_flush_local_cache(mr_ctrl); 607 /* Linear search on MR cache array. */ 608 lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru, 609 MLX5_MR_CACHE_N, addr); 610 if (likely(lkey != UINT32_MAX)) 611 return lkey; 612 /* Take slower bottom-half on miss. */ 613 return mlx5_tx_mb2mr_bh(txq, mb); 614 } 615 616 /** 617 * Ring TX queue doorbell and flush the update if requested. 618 * 619 * @param txq 620 * Pointer to TX queue structure. 621 * @param wqe 622 * Pointer to the last WQE posted in the NIC. 623 * @param cond 624 * Request for write memory barrier after BlueFlame update. 625 */ 626 static __rte_always_inline void 627 mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe, 628 int cond) 629 { 630 uint64_t *dst = MLX5_TX_BFREG(txq); 631 volatile uint64_t *src = ((volatile uint64_t *)wqe); 632 633 rte_cio_wmb(); 634 *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci); 635 /* Ensure ordering between DB record and BF copy. */ 636 rte_wmb(); 637 mlx5_uar_write64_relaxed(*src, dst, txq->uar_lock); 638 if (cond) 639 rte_wmb(); 640 } 641 642 /** 643 * Ring TX queue doorbell and flush the update by write memory barrier. 644 * 645 * @param txq 646 * Pointer to TX queue structure. 647 * @param wqe 648 * Pointer to the last WQE posted in the NIC. 649 */ 650 static __rte_always_inline void 651 mlx5_tx_dbrec(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe) 652 { 653 mlx5_tx_dbrec_cond_wmb(txq, wqe, 1); 654 } 655 656 #endif /* RTE_PMD_MLX5_RXTX_H_ */ 657