1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_malloc.h> 19 20 #include "mlx5.h" 21 #include "mlx5_common_os.h" 22 #include "mlx5_rxtx.h" 23 #include "mlx5_utils.h" 24 #include "mlx5_devx.h" 25 #include "mlx5_flow.h" 26 #include "mlx5_flow_os.h" 27 28 /** 29 * Modify RQ vlan stripping offload 30 * 31 * @param rxq_obj 32 * Rx queue object. 33 * 34 * @return 35 * 0 on success, non-0 otherwise 36 */ 37 static int 38 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_obj *rxq_obj, int on) 39 { 40 struct mlx5_devx_modify_rq_attr rq_attr; 41 42 memset(&rq_attr, 0, sizeof(rq_attr)); 43 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 44 rq_attr.state = MLX5_RQC_STATE_RDY; 45 rq_attr.vsd = (on ? 0 : 1); 46 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 47 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 48 } 49 50 /** 51 * Modify RQ using DevX API. 52 * 53 * @param rxq_obj 54 * DevX Rx queue object. 55 * @param type 56 * Type of change queue state. 57 * 58 * @return 59 * 0 on success, a negative errno value otherwise and rte_errno is set. 60 */ 61 static int 62 mlx5_devx_modify_rq(struct mlx5_rxq_obj *rxq_obj, uint8_t type) 63 { 64 struct mlx5_devx_modify_rq_attr rq_attr; 65 66 memset(&rq_attr, 0, sizeof(rq_attr)); 67 switch (type) { 68 case MLX5_RXQ_MOD_ERR2RST: 69 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 70 rq_attr.state = MLX5_RQC_STATE_RST; 71 break; 72 case MLX5_RXQ_MOD_RST2RDY: 73 rq_attr.rq_state = MLX5_RQC_STATE_RST; 74 rq_attr.state = MLX5_RQC_STATE_RDY; 75 break; 76 case MLX5_RXQ_MOD_RDY2ERR: 77 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 78 rq_attr.state = MLX5_RQC_STATE_ERR; 79 break; 80 case MLX5_RXQ_MOD_RDY2RST: 81 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 82 rq_attr.state = MLX5_RQC_STATE_RST; 83 break; 84 default: 85 break; 86 } 87 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 88 } 89 90 /** 91 * Modify SQ using DevX API. 92 * 93 * @param txq_obj 94 * DevX Tx queue object. 95 * @param type 96 * Type of change queue state. 97 * @param dev_port 98 * Unnecessary. 99 * 100 * @return 101 * 0 on success, a negative errno value otherwise and rte_errno is set. 102 */ 103 static int 104 mlx5_devx_modify_sq(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type, 105 uint8_t dev_port) 106 { 107 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 108 int ret; 109 110 if (type != MLX5_TXQ_MOD_RST2RDY) { 111 /* Change queue state to reset. */ 112 if (type == MLX5_TXQ_MOD_ERR2RDY) 113 msq_attr.sq_state = MLX5_SQC_STATE_ERR; 114 else 115 msq_attr.sq_state = MLX5_SQC_STATE_RDY; 116 msq_attr.state = MLX5_SQC_STATE_RST; 117 ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr); 118 if (ret) { 119 DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET" 120 " %s", strerror(errno)); 121 rte_errno = errno; 122 return ret; 123 } 124 } 125 if (type != MLX5_TXQ_MOD_RDY2RST) { 126 /* Change queue state to ready. */ 127 msq_attr.sq_state = MLX5_SQC_STATE_RST; 128 msq_attr.state = MLX5_SQC_STATE_RDY; 129 ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr); 130 if (ret) { 131 DRV_LOG(ERR, "Cannot change the Tx SQ state to READY" 132 " %s", strerror(errno)); 133 rte_errno = errno; 134 return ret; 135 } 136 } 137 /* 138 * The dev_port variable is relevant only in Verbs API, and there is a 139 * pointer that points to this function and a parallel function in verbs 140 * intermittently, so they should have the same parameters. 141 */ 142 (void)dev_port; 143 return 0; 144 } 145 146 /** 147 * Release the resources allocated for an RQ DevX object. 148 * 149 * @param rxq_ctrl 150 * DevX Rx queue object. 151 */ 152 static void 153 mlx5_rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 154 { 155 struct mlx5_devx_dbr_page *dbr_page = rxq_ctrl->rq_dbrec_page; 156 157 if (rxq_ctrl->wq_umem) { 158 mlx5_os_umem_dereg(rxq_ctrl->wq_umem); 159 rxq_ctrl->wq_umem = NULL; 160 } 161 if (rxq_ctrl->rxq.wqes) { 162 mlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes); 163 rxq_ctrl->rxq.wqes = NULL; 164 } 165 if (dbr_page) { 166 claim_zero(mlx5_release_dbr(&rxq_ctrl->priv->dbrpgs, 167 mlx5_os_get_umem_id(dbr_page->umem), 168 rxq_ctrl->rq_dbr_offset)); 169 rxq_ctrl->rq_dbrec_page = NULL; 170 } 171 } 172 173 /** 174 * Release the resources allocated for the Rx CQ DevX object. 175 * 176 * @param rxq_ctrl 177 * DevX Rx queue object. 178 */ 179 static void 180 mlx5_rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 181 { 182 struct mlx5_devx_dbr_page *dbr_page = rxq_ctrl->cq_dbrec_page; 183 184 if (rxq_ctrl->cq_umem) { 185 mlx5_os_umem_dereg(rxq_ctrl->cq_umem); 186 rxq_ctrl->cq_umem = NULL; 187 } 188 if (rxq_ctrl->rxq.cqes) { 189 rte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes); 190 rxq_ctrl->rxq.cqes = NULL; 191 } 192 if (dbr_page) { 193 claim_zero(mlx5_release_dbr(&rxq_ctrl->priv->dbrpgs, 194 mlx5_os_get_umem_id(dbr_page->umem), 195 rxq_ctrl->cq_dbr_offset)); 196 rxq_ctrl->cq_dbrec_page = NULL; 197 } 198 } 199 200 /** 201 * Release an Rx DevX queue object. 202 * 203 * @param rxq_obj 204 * DevX Rx queue object. 205 */ 206 static void 207 mlx5_rxq_devx_obj_release(struct mlx5_rxq_obj *rxq_obj) 208 { 209 MLX5_ASSERT(rxq_obj); 210 MLX5_ASSERT(rxq_obj->rq); 211 if (rxq_obj->rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 212 mlx5_devx_modify_rq(rxq_obj, MLX5_RXQ_MOD_RDY2RST); 213 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 214 } else { 215 MLX5_ASSERT(rxq_obj->devx_cq); 216 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 217 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq)); 218 if (rxq_obj->devx_channel) 219 mlx5_os_devx_destroy_event_channel 220 (rxq_obj->devx_channel); 221 mlx5_rxq_release_devx_rq_resources(rxq_obj->rxq_ctrl); 222 mlx5_rxq_release_devx_cq_resources(rxq_obj->rxq_ctrl); 223 } 224 } 225 226 /** 227 * Get event for an Rx DevX queue object. 228 * 229 * @param rxq_obj 230 * DevX Rx queue object. 231 * 232 * @return 233 * 0 on success, a negative errno value otherwise and rte_errno is set. 234 */ 235 static int 236 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 237 { 238 #ifdef HAVE_IBV_DEVX_EVENT 239 union { 240 struct mlx5dv_devx_async_event_hdr event_resp; 241 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 242 } out; 243 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 244 &out.event_resp, 245 sizeof(out.buf)); 246 247 if (ret < 0) { 248 rte_errno = errno; 249 return -rte_errno; 250 } 251 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->devx_cq) { 252 rte_errno = EINVAL; 253 return -rte_errno; 254 } 255 return 0; 256 #else 257 (void)rxq_obj; 258 rte_errno = ENOTSUP; 259 return -rte_errno; 260 #endif /* HAVE_IBV_DEVX_EVENT */ 261 } 262 263 /** 264 * Fill common fields of create RQ attributes structure. 265 * 266 * @param rxq_data 267 * Pointer to Rx queue data. 268 * @param cqn 269 * CQ number to use with this RQ. 270 * @param rq_attr 271 * RQ attributes structure to fill.. 272 */ 273 static void 274 mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn, 275 struct mlx5_devx_create_rq_attr *rq_attr) 276 { 277 rq_attr->state = MLX5_RQC_STATE_RST; 278 rq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1; 279 rq_attr->cqn = cqn; 280 rq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 281 } 282 283 /** 284 * Fill common fields of DevX WQ attributes structure. 285 * 286 * @param priv 287 * Pointer to device private data. 288 * @param rxq_ctrl 289 * Pointer to Rx queue control structure. 290 * @param wq_attr 291 * WQ attributes structure to fill.. 292 */ 293 static void 294 mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl, 295 struct mlx5_devx_wq_attr *wq_attr) 296 { 297 wq_attr->end_padding_mode = priv->config.hw_padding ? 298 MLX5_WQ_END_PAD_MODE_ALIGN : 299 MLX5_WQ_END_PAD_MODE_NONE; 300 wq_attr->pd = priv->sh->pdn; 301 wq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset; 302 wq_attr->dbr_umem_id = 303 mlx5_os_get_umem_id(rxq_ctrl->rq_dbrec_page->umem); 304 wq_attr->dbr_umem_valid = 1; 305 wq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem); 306 wq_attr->wq_umem_valid = 1; 307 } 308 309 /** 310 * Create a RQ object using DevX. 311 * 312 * @param dev 313 * Pointer to Ethernet device. 314 * @param idx 315 * Queue index in DPDK Rx queue array. 316 * 317 * @return 318 * The DevX RQ object initialized, NULL otherwise and rte_errno is set. 319 */ 320 static struct mlx5_devx_obj * 321 mlx5_rxq_create_devx_rq_resources(struct rte_eth_dev *dev, uint16_t idx) 322 { 323 struct mlx5_priv *priv = dev->data->dev_private; 324 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 325 struct mlx5_rxq_ctrl *rxq_ctrl = 326 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 327 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 328 uint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n); 329 uint32_t cqn = rxq_ctrl->obj->devx_cq->id; 330 struct mlx5_devx_dbr_page *dbr_page; 331 int64_t dbr_offset; 332 uint32_t wq_size = 0; 333 uint32_t wqe_size = 0; 334 uint32_t log_wqe_size = 0; 335 void *buf = NULL; 336 struct mlx5_devx_obj *rq; 337 338 /* Fill RQ attributes. */ 339 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 340 rq_attr.flush_in_error_en = 1; 341 mlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr); 342 /* Fill WQ attributes for this RQ. */ 343 if (mlx5_rxq_mprq_enabled(rxq_data)) { 344 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 345 /* 346 * Number of strides in each WQE: 347 * 512*2^single_wqe_log_num_of_strides. 348 */ 349 rq_attr.wq_attr.single_wqe_log_num_of_strides = 350 rxq_data->strd_num_n - 351 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 352 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 353 rq_attr.wq_attr.single_stride_log_num_of_bytes = 354 rxq_data->strd_sz_n - 355 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 356 wqe_size = sizeof(struct mlx5_wqe_mprq); 357 } else { 358 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 359 wqe_size = sizeof(struct mlx5_wqe_data_seg); 360 } 361 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 362 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 363 rq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n; 364 /* Calculate and allocate WQ memory space. */ 365 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 366 wq_size = wqe_n * wqe_size; 367 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 368 if (alignment == (size_t)-1) { 369 DRV_LOG(ERR, "Failed to get mem page size"); 370 rte_errno = ENOMEM; 371 return NULL; 372 } 373 buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size, 374 alignment, rxq_ctrl->socket); 375 if (!buf) 376 return NULL; 377 rxq_data->wqes = buf; 378 rxq_ctrl->wq_umem = mlx5_os_umem_reg(priv->sh->ctx, 379 buf, wq_size, 0); 380 if (!rxq_ctrl->wq_umem) 381 goto error; 382 /* Allocate RQ door-bell. */ 383 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &dbr_page); 384 if (dbr_offset < 0) { 385 DRV_LOG(ERR, "Failed to allocate RQ door-bell."); 386 goto error; 387 } 388 rxq_ctrl->rq_dbr_offset = dbr_offset; 389 rxq_ctrl->rq_dbrec_page = dbr_page; 390 rxq_data->rq_db = (uint32_t *)((uintptr_t)dbr_page->dbrs + 391 (uintptr_t)rxq_ctrl->rq_dbr_offset); 392 /* Create RQ using DevX API. */ 393 mlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr); 394 rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket); 395 if (!rq) 396 goto error; 397 return rq; 398 error: 399 mlx5_rxq_release_devx_rq_resources(rxq_ctrl); 400 return NULL; 401 } 402 403 /** 404 * Create a DevX CQ object for an Rx queue. 405 * 406 * @param dev 407 * Pointer to Ethernet device. 408 * @param idx 409 * Queue index in DPDK Rx queue array. 410 * 411 * @return 412 * The DevX CQ object initialized, NULL otherwise and rte_errno is set. 413 */ 414 static struct mlx5_devx_obj * 415 mlx5_rxq_create_devx_cq_resources(struct rte_eth_dev *dev, uint16_t idx) 416 { 417 struct mlx5_devx_obj *cq_obj = 0; 418 struct mlx5_devx_cq_attr cq_attr = { 0 }; 419 struct mlx5_priv *priv = dev->data->dev_private; 420 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 421 struct mlx5_rxq_ctrl *rxq_ctrl = 422 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 423 size_t page_size = rte_mem_page_size(); 424 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 425 struct mlx5_devx_dbr_page *dbr_page; 426 int64_t dbr_offset; 427 void *buf = NULL; 428 uint16_t event_nums[1] = {0}; 429 uint32_t log_cqe_n; 430 uint32_t cq_size; 431 int ret = 0; 432 433 if (page_size == (size_t)-1) { 434 DRV_LOG(ERR, "Failed to get page_size."); 435 goto error; 436 } 437 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 438 !rxq_data->lro) { 439 cq_attr.cqe_comp_en = 1u; 440 rxq_data->mcqe_format = priv->config.cqe_comp_fmt; 441 rxq_data->byte_mask = UINT32_MAX; 442 switch (priv->config.cqe_comp_fmt) { 443 case MLX5_CQE_RESP_FORMAT_HASH: 444 /* fallthrough */ 445 case MLX5_CQE_RESP_FORMAT_CSUM: 446 /* 447 * Select CSUM miniCQE format only for non-vectorized 448 * MPRQ Rx burst, use HASH miniCQE format for others. 449 */ 450 if (mlx5_rxq_check_vec_support(rxq_data) < 0 && 451 mlx5_rxq_mprq_enabled(rxq_data)) 452 cq_attr.mini_cqe_res_format = 453 MLX5_CQE_RESP_FORMAT_CSUM_STRIDX; 454 else 455 cq_attr.mini_cqe_res_format = 456 MLX5_CQE_RESP_FORMAT_HASH; 457 rxq_data->mcqe_format = cq_attr.mini_cqe_res_format; 458 break; 459 case MLX5_CQE_RESP_FORMAT_FTAG_STRIDX: 460 rxq_data->byte_mask = MLX5_LEN_WITH_MARK_MASK; 461 /* fallthrough */ 462 case MLX5_CQE_RESP_FORMAT_CSUM_STRIDX: 463 cq_attr.mini_cqe_res_format = priv->config.cqe_comp_fmt; 464 break; 465 case MLX5_CQE_RESP_FORMAT_L34H_STRIDX: 466 cq_attr.mini_cqe_res_format = 0; 467 cq_attr.mini_cqe_res_format_ext = 1; 468 break; 469 } 470 DRV_LOG(DEBUG, 471 "Port %u Rx CQE compression is enabled, format %d.", 472 dev->data->port_id, priv->config.cqe_comp_fmt); 473 /* 474 * For vectorized Rx, it must not be doubled in order to 475 * make cq_ci and rq_ci aligned. 476 */ 477 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 478 cqe_n *= 2; 479 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 480 DRV_LOG(DEBUG, 481 "Port %u Rx CQE compression is disabled for HW" 482 " timestamp.", 483 dev->data->port_id); 484 } else if (priv->config.cqe_comp && rxq_data->lro) { 485 DRV_LOG(DEBUG, 486 "Port %u Rx CQE compression is disabled for LRO.", 487 dev->data->port_id); 488 } 489 if (priv->config.cqe_pad) 490 cq_attr.cqe_size = MLX5_CQE_SIZE_128B; 491 log_cqe_n = log2above(cqe_n); 492 cq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n); 493 buf = rte_calloc_socket(__func__, 1, cq_size, page_size, 494 rxq_ctrl->socket); 495 if (!buf) { 496 DRV_LOG(ERR, "Failed to allocate memory for CQ."); 497 goto error; 498 } 499 rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf; 500 rxq_ctrl->cq_umem = mlx5_os_umem_reg(priv->sh->ctx, buf, 501 cq_size, 502 IBV_ACCESS_LOCAL_WRITE); 503 if (!rxq_ctrl->cq_umem) { 504 DRV_LOG(ERR, "Failed to register umem for CQ."); 505 goto error; 506 } 507 /* Allocate CQ door-bell. */ 508 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &dbr_page); 509 if (dbr_offset < 0) { 510 DRV_LOG(ERR, "Failed to allocate CQ door-bell."); 511 goto error; 512 } 513 rxq_ctrl->cq_dbr_offset = dbr_offset; 514 rxq_ctrl->cq_dbrec_page = dbr_page; 515 rxq_data->cq_db = (uint32_t *)((uintptr_t)dbr_page->dbrs + 516 (uintptr_t)rxq_ctrl->cq_dbr_offset); 517 rxq_data->cq_uar = 518 mlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar); 519 /* Create CQ using DevX API. */ 520 cq_attr.eqn = priv->sh->eqn; 521 cq_attr.uar_page_id = 522 mlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar); 523 cq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem); 524 cq_attr.q_umem_valid = 1; 525 cq_attr.log_cq_size = log_cqe_n; 526 cq_attr.log_page_size = rte_log2_u32(page_size); 527 cq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset; 528 cq_attr.db_umem_id = mlx5_os_get_umem_id(dbr_page->umem); 529 cq_attr.db_umem_valid = 1; 530 cq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr); 531 if (!cq_obj) 532 goto error; 533 rxq_data->cqe_n = log_cqe_n; 534 rxq_data->cqn = cq_obj->id; 535 if (rxq_ctrl->obj->devx_channel) { 536 ret = mlx5_os_devx_subscribe_devx_event 537 (rxq_ctrl->obj->devx_channel, 538 cq_obj->obj, 539 sizeof(event_nums), 540 event_nums, 541 (uint64_t)(uintptr_t)cq_obj); 542 if (ret) { 543 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 544 rte_errno = errno; 545 goto error; 546 } 547 } 548 /* Initialise CQ to 1's to mark HW ownership for all CQEs. */ 549 memset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size); 550 return cq_obj; 551 error: 552 if (cq_obj) 553 mlx5_devx_cmd_destroy(cq_obj); 554 mlx5_rxq_release_devx_cq_resources(rxq_ctrl); 555 return NULL; 556 } 557 558 /** 559 * Create the Rx hairpin queue object. 560 * 561 * @param dev 562 * Pointer to Ethernet device. 563 * @param idx 564 * Queue index in DPDK Rx queue array. 565 * 566 * @return 567 * 0 on success, a negative errno value otherwise and rte_errno is set. 568 */ 569 static int 570 mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 571 { 572 struct mlx5_priv *priv = dev->data->dev_private; 573 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 574 struct mlx5_rxq_ctrl *rxq_ctrl = 575 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 576 struct mlx5_devx_create_rq_attr attr = { 0 }; 577 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 578 uint32_t max_wq_data; 579 580 MLX5_ASSERT(rxq_data); 581 MLX5_ASSERT(tmpl); 582 tmpl->rxq_ctrl = rxq_ctrl; 583 attr.hairpin = 1; 584 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 585 /* Jumbo frames > 9KB should be supported, and more packets. */ 586 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 587 if (priv->config.log_hp_size > max_wq_data) { 588 DRV_LOG(ERR, "Total data size %u power of 2 is " 589 "too large for hairpin.", 590 priv->config.log_hp_size); 591 rte_errno = ERANGE; 592 return -rte_errno; 593 } 594 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 595 } else { 596 attr.wq_attr.log_hairpin_data_sz = 597 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 598 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 599 } 600 /* Set the packets number to the maximum value for performance. */ 601 attr.wq_attr.log_hairpin_num_packets = 602 attr.wq_attr.log_hairpin_data_sz - 603 MLX5_HAIRPIN_QUEUE_STRIDE; 604 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr, 605 rxq_ctrl->socket); 606 if (!tmpl->rq) { 607 DRV_LOG(ERR, 608 "Port %u Rx hairpin queue %u can't create rq object.", 609 dev->data->port_id, idx); 610 rte_errno = errno; 611 return -rte_errno; 612 } 613 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 614 return 0; 615 } 616 617 /** 618 * Create the Rx queue DevX object. 619 * 620 * @param dev 621 * Pointer to Ethernet device. 622 * @param idx 623 * Queue index in DPDK Rx queue array. 624 * 625 * @return 626 * 0 on success, a negative errno value otherwise and rte_errno is set. 627 */ 628 static int 629 mlx5_rxq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 630 { 631 struct mlx5_priv *priv = dev->data->dev_private; 632 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 633 struct mlx5_rxq_ctrl *rxq_ctrl = 634 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 635 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 636 int ret = 0; 637 638 MLX5_ASSERT(rxq_data); 639 MLX5_ASSERT(tmpl); 640 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 641 return mlx5_rxq_obj_hairpin_new(dev, idx); 642 tmpl->rxq_ctrl = rxq_ctrl; 643 if (rxq_ctrl->irq) { 644 int devx_ev_flag = 645 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 646 647 tmpl->devx_channel = mlx5_os_devx_create_event_channel 648 (priv->sh->ctx, 649 devx_ev_flag); 650 if (!tmpl->devx_channel) { 651 rte_errno = errno; 652 DRV_LOG(ERR, "Failed to create event channel %d.", 653 rte_errno); 654 goto error; 655 } 656 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 657 } 658 /* Create CQ using DevX API. */ 659 tmpl->devx_cq = mlx5_rxq_create_devx_cq_resources(dev, idx); 660 if (!tmpl->devx_cq) { 661 DRV_LOG(ERR, "Failed to create CQ."); 662 goto error; 663 } 664 /* Create RQ using DevX API. */ 665 tmpl->rq = mlx5_rxq_create_devx_rq_resources(dev, idx); 666 if (!tmpl->rq) { 667 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 668 dev->data->port_id, idx); 669 rte_errno = ENOMEM; 670 goto error; 671 } 672 /* Change queue state to ready. */ 673 ret = mlx5_devx_modify_rq(tmpl, MLX5_RXQ_MOD_RST2RDY); 674 if (ret) 675 goto error; 676 rxq_data->cq_arm_sn = 0; 677 mlx5_rxq_initialize(rxq_data); 678 rxq_data->cq_ci = 0; 679 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 680 rxq_ctrl->wqn = tmpl->rq->id; 681 return 0; 682 error: 683 ret = rte_errno; /* Save rte_errno before cleanup. */ 684 if (tmpl->rq) 685 claim_zero(mlx5_devx_cmd_destroy(tmpl->rq)); 686 if (tmpl->devx_cq) 687 claim_zero(mlx5_devx_cmd_destroy(tmpl->devx_cq)); 688 if (tmpl->devx_channel) 689 mlx5_os_devx_destroy_event_channel(tmpl->devx_channel); 690 mlx5_rxq_release_devx_rq_resources(rxq_ctrl); 691 mlx5_rxq_release_devx_cq_resources(rxq_ctrl); 692 rte_errno = ret; /* Restore rte_errno. */ 693 return -rte_errno; 694 } 695 696 /** 697 * Prepare RQT attribute structure for DevX RQT API. 698 * 699 * @param dev 700 * Pointer to Ethernet device. 701 * @param log_n 702 * Log of number of queues in the array. 703 * @param ind_tbl 704 * DevX indirection table object. 705 * 706 * @return 707 * The RQT attr object initialized, NULL otherwise and rte_errno is set. 708 */ 709 static struct mlx5_devx_rqt_attr * 710 mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev, 711 const unsigned int log_n, 712 const uint16_t *queues, 713 const uint32_t queues_n) 714 { 715 struct mlx5_priv *priv = dev->data->dev_private; 716 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 717 const unsigned int rqt_n = 1 << log_n; 718 unsigned int i, j; 719 720 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 721 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 722 if (!rqt_attr) { 723 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 724 dev->data->port_id); 725 rte_errno = ENOMEM; 726 return NULL; 727 } 728 rqt_attr->rqt_max_size = priv->config.ind_table_max_size; 729 rqt_attr->rqt_actual_size = rqt_n; 730 for (i = 0; i != queues_n; ++i) { 731 struct mlx5_rxq_data *rxq = (*priv->rxqs)[queues[i]]; 732 struct mlx5_rxq_ctrl *rxq_ctrl = 733 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 734 735 rqt_attr->rq_list[i] = rxq_ctrl->obj->rq->id; 736 } 737 MLX5_ASSERT(i > 0); 738 for (j = 0; i != rqt_n; ++j, ++i) 739 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 740 return rqt_attr; 741 } 742 743 /** 744 * Create RQT using DevX API as a filed of indirection table. 745 * 746 * @param dev 747 * Pointer to Ethernet device. 748 * @param log_n 749 * Log of number of queues in the array. 750 * @param ind_tbl 751 * DevX indirection table object. 752 * 753 * @return 754 * 0 on success, a negative errno value otherwise and rte_errno is set. 755 */ 756 static int 757 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 758 struct mlx5_ind_table_obj *ind_tbl) 759 { 760 struct mlx5_priv *priv = dev->data->dev_private; 761 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 762 763 MLX5_ASSERT(ind_tbl); 764 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, 765 ind_tbl->queues, 766 ind_tbl->queues_n); 767 if (!rqt_attr) 768 return -rte_errno; 769 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->ctx, rqt_attr); 770 mlx5_free(rqt_attr); 771 if (!ind_tbl->rqt) { 772 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 773 dev->data->port_id); 774 rte_errno = errno; 775 return -rte_errno; 776 } 777 return 0; 778 } 779 780 /** 781 * Modify RQT using DevX API as a filed of indirection table. 782 * 783 * @param dev 784 * Pointer to Ethernet device. 785 * @param log_n 786 * Log of number of queues in the array. 787 * @param ind_tbl 788 * DevX indirection table object. 789 * 790 * @return 791 * 0 on success, a negative errno value otherwise and rte_errno is set. 792 */ 793 static int 794 mlx5_devx_ind_table_modify(struct rte_eth_dev *dev, const unsigned int log_n, 795 const uint16_t *queues, const uint32_t queues_n, 796 struct mlx5_ind_table_obj *ind_tbl) 797 { 798 int ret = 0; 799 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 800 801 MLX5_ASSERT(ind_tbl); 802 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, 803 queues, 804 queues_n); 805 if (!rqt_attr) 806 return -rte_errno; 807 ret = mlx5_devx_cmd_modify_rqt(ind_tbl->rqt, rqt_attr); 808 mlx5_free(rqt_attr); 809 if (ret) 810 DRV_LOG(ERR, "Port %u cannot modify DevX RQT.", 811 dev->data->port_id); 812 return ret; 813 } 814 815 /** 816 * Destroy the DevX RQT object. 817 * 818 * @param ind_table 819 * Indirection table to release. 820 */ 821 static void 822 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 823 { 824 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 825 } 826 827 /** 828 * Set TIR attribute struct with relevant input values. 829 * 830 * @param[in] dev 831 * Pointer to Ethernet device. 832 * @param[in] rss_key 833 * RSS key for the Rx hash queue. 834 * @param[in] hash_fields 835 * Verbs protocol hash field to make the RSS on. 836 * @param[in] ind_tbl 837 * Indirection table for TIR. 838 * @param[in] tunnel 839 * Tunnel type. 840 * @param[out] tir_attr 841 * Parameters structure for TIR creation/modification. 842 * 843 * @return 844 * The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set. 845 */ 846 static void 847 mlx5_devx_tir_attr_set(struct rte_eth_dev *dev, const uint8_t *rss_key, 848 uint64_t hash_fields, 849 const struct mlx5_ind_table_obj *ind_tbl, 850 int tunnel, struct mlx5_devx_tir_attr *tir_attr) 851 { 852 struct mlx5_priv *priv = dev->data->dev_private; 853 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[ind_tbl->queues[0]]; 854 struct mlx5_rxq_ctrl *rxq_ctrl = 855 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 856 enum mlx5_rxq_type rxq_obj_type = rxq_ctrl->type; 857 bool lro = true; 858 uint32_t i; 859 860 /* Enable TIR LRO only if all the queues were configured for. */ 861 for (i = 0; i < ind_tbl->queues_n; ++i) { 862 if (!(*priv->rxqs)[ind_tbl->queues[i]]->lro) { 863 lro = false; 864 break; 865 } 866 } 867 memset(tir_attr, 0, sizeof(*tir_attr)); 868 tir_attr->disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 869 tir_attr->rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 870 tir_attr->tunneled_offload_en = !!tunnel; 871 /* If needed, translate hash_fields bitmap to PRM format. */ 872 if (hash_fields) { 873 struct mlx5_rx_hash_field_select *rx_hash_field_select = 874 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 875 hash_fields & IBV_RX_HASH_INNER ? 876 &tir_attr->rx_hash_field_selector_inner : 877 #endif 878 &tir_attr->rx_hash_field_selector_outer; 879 /* 1 bit: 0: IPv4, 1: IPv6. */ 880 rx_hash_field_select->l3_prot_type = 881 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 882 /* 1 bit: 0: TCP, 1: UDP. */ 883 rx_hash_field_select->l4_prot_type = 884 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 885 /* Bitmask which sets which fields to use in RX Hash. */ 886 rx_hash_field_select->selected_fields = 887 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 888 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 889 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 890 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 891 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 892 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 893 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 894 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT; 895 } 896 if (rxq_obj_type == MLX5_RXQ_TYPE_HAIRPIN) 897 tir_attr->transport_domain = priv->sh->td->id; 898 else 899 tir_attr->transport_domain = priv->sh->tdn; 900 memcpy(tir_attr->rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 901 tir_attr->indirect_table = ind_tbl->rqt->id; 902 if (dev->data->dev_conf.lpbk_mode) 903 tir_attr->self_lb_block = 904 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 905 if (lro) { 906 tir_attr->lro_timeout_period_usecs = priv->config.lro.timeout; 907 tir_attr->lro_max_msg_sz = priv->max_lro_msg_size; 908 tir_attr->lro_enable_mask = 909 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 910 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 911 } 912 } 913 914 /** 915 * Create an Rx Hash queue. 916 * 917 * @param dev 918 * Pointer to Ethernet device. 919 * @param hrxq 920 * Pointer to Rx Hash queue. 921 * @param tunnel 922 * Tunnel type. 923 * 924 * @return 925 * 0 on success, a negative errno value otherwise and rte_errno is set. 926 */ 927 static int 928 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 929 int tunnel __rte_unused) 930 { 931 struct mlx5_priv *priv = dev->data->dev_private; 932 struct mlx5_devx_tir_attr tir_attr = {0}; 933 int err; 934 935 mlx5_devx_tir_attr_set(dev, hrxq->rss_key, hrxq->hash_fields, 936 hrxq->ind_table, tunnel, &tir_attr); 937 hrxq->tir = mlx5_devx_cmd_create_tir(priv->sh->ctx, &tir_attr); 938 if (!hrxq->tir) { 939 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 940 dev->data->port_id); 941 rte_errno = errno; 942 goto error; 943 } 944 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 945 if (mlx5_flow_os_create_flow_action_dest_devx_tir(hrxq->tir, 946 &hrxq->action)) { 947 rte_errno = errno; 948 goto error; 949 } 950 #endif 951 return 0; 952 error: 953 err = rte_errno; /* Save rte_errno before cleanup. */ 954 if (hrxq->tir) 955 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 956 rte_errno = err; /* Restore rte_errno. */ 957 return -rte_errno; 958 } 959 960 /** 961 * Destroy a DevX TIR object. 962 * 963 * @param hrxq 964 * Hash Rx queue to release its tir. 965 */ 966 static void 967 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 968 { 969 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 970 } 971 972 /** 973 * Modify an Rx Hash queue configuration. 974 * 975 * @param dev 976 * Pointer to Ethernet device. 977 * @param hrxq 978 * Hash Rx queue to modify. 979 * @param rss_key 980 * RSS key for the Rx hash queue. 981 * @param hash_fields 982 * Verbs protocol hash field to make the RSS on. 983 * @param[in] ind_tbl 984 * Indirection table for TIR. 985 * 986 * @return 987 * 0 on success, a negative errno value otherwise and rte_errno is set. 988 */ 989 static int 990 mlx5_devx_hrxq_modify(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 991 const uint8_t *rss_key, 992 uint64_t hash_fields, 993 const struct mlx5_ind_table_obj *ind_tbl) 994 { 995 struct mlx5_devx_modify_tir_attr modify_tir = {0}; 996 997 /* 998 * untested for modification fields: 999 * - rx_hash_symmetric not set in hrxq_new(), 1000 * - rx_hash_fn set hard-coded in hrxq_new(), 1001 * - lro_xxx not set after rxq setup 1002 */ 1003 if (ind_tbl != hrxq->ind_table) 1004 modify_tir.modify_bitmask |= 1005 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_INDIRECT_TABLE; 1006 if (hash_fields != hrxq->hash_fields || 1007 memcmp(hrxq->rss_key, rss_key, MLX5_RSS_HASH_KEY_LEN)) 1008 modify_tir.modify_bitmask |= 1009 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_HASH; 1010 mlx5_devx_tir_attr_set(dev, rss_key, hash_fields, ind_tbl, 1011 0, /* N/A - tunnel modification unsupported */ 1012 &modify_tir.tir); 1013 modify_tir.tirn = hrxq->tir->id; 1014 if (mlx5_devx_cmd_modify_tir(hrxq->tir, &modify_tir)) { 1015 DRV_LOG(ERR, "port %u cannot modify DevX TIR", 1016 dev->data->port_id); 1017 rte_errno = errno; 1018 return -rte_errno; 1019 } 1020 return 0; 1021 } 1022 1023 /** 1024 * Create a DevX drop action for Rx Hash queue. 1025 * 1026 * @param dev 1027 * Pointer to Ethernet device. 1028 * 1029 * @return 1030 * 0 on success, a negative errno value otherwise and rte_errno is set. 1031 */ 1032 static int 1033 mlx5_devx_drop_action_create(struct rte_eth_dev *dev) 1034 { 1035 (void)dev; 1036 DRV_LOG(ERR, "DevX drop action is not supported yet."); 1037 rte_errno = ENOTSUP; 1038 return -rte_errno; 1039 } 1040 1041 /** 1042 * Release a drop hash Rx queue. 1043 * 1044 * @param dev 1045 * Pointer to Ethernet device. 1046 */ 1047 static void 1048 mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) 1049 { 1050 (void)dev; 1051 DRV_LOG(ERR, "DevX drop action is not supported yet."); 1052 rte_errno = ENOTSUP; 1053 } 1054 1055 /** 1056 * Create the Tx hairpin queue object. 1057 * 1058 * @param dev 1059 * Pointer to Ethernet device. 1060 * @param idx 1061 * Queue index in DPDK Tx queue array. 1062 * 1063 * @return 1064 * 0 on success, a negative errno value otherwise and rte_errno is set. 1065 */ 1066 static int 1067 mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 1068 { 1069 struct mlx5_priv *priv = dev->data->dev_private; 1070 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1071 struct mlx5_txq_ctrl *txq_ctrl = 1072 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1073 struct mlx5_devx_create_sq_attr attr = { 0 }; 1074 struct mlx5_txq_obj *tmpl = txq_ctrl->obj; 1075 uint32_t max_wq_data; 1076 1077 MLX5_ASSERT(txq_data); 1078 MLX5_ASSERT(tmpl); 1079 tmpl->txq_ctrl = txq_ctrl; 1080 attr.hairpin = 1; 1081 attr.tis_lst_sz = 1; 1082 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 1083 /* Jumbo frames > 9KB should be supported, and more packets. */ 1084 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 1085 if (priv->config.log_hp_size > max_wq_data) { 1086 DRV_LOG(ERR, "Total data size %u power of 2 is " 1087 "too large for hairpin.", 1088 priv->config.log_hp_size); 1089 rte_errno = ERANGE; 1090 return -rte_errno; 1091 } 1092 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 1093 } else { 1094 attr.wq_attr.log_hairpin_data_sz = 1095 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 1096 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 1097 } 1098 /* Set the packets number to the maximum value for performance. */ 1099 attr.wq_attr.log_hairpin_num_packets = 1100 attr.wq_attr.log_hairpin_data_sz - 1101 MLX5_HAIRPIN_QUEUE_STRIDE; 1102 attr.tis_num = priv->sh->tis->id; 1103 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->ctx, &attr); 1104 if (!tmpl->sq) { 1105 DRV_LOG(ERR, 1106 "Port %u tx hairpin queue %u can't create SQ object.", 1107 dev->data->port_id, idx); 1108 rte_errno = errno; 1109 return -rte_errno; 1110 } 1111 return 0; 1112 } 1113 1114 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1115 /** 1116 * Release DevX SQ resources. 1117 * 1118 * @param txq_obj 1119 * DevX Tx queue object. 1120 */ 1121 static void 1122 mlx5_txq_release_devx_sq_resources(struct mlx5_txq_obj *txq_obj) 1123 { 1124 if (txq_obj->sq_devx) { 1125 claim_zero(mlx5_devx_cmd_destroy(txq_obj->sq_devx)); 1126 txq_obj->sq_devx = NULL; 1127 } 1128 if (txq_obj->sq_umem) { 1129 claim_zero(mlx5_os_umem_dereg(txq_obj->sq_umem)); 1130 txq_obj->sq_umem = NULL; 1131 } 1132 if (txq_obj->sq_buf) { 1133 mlx5_free(txq_obj->sq_buf); 1134 txq_obj->sq_buf = NULL; 1135 } 1136 if (txq_obj->sq_dbrec_page) { 1137 claim_zero(mlx5_release_dbr(&txq_obj->txq_ctrl->priv->dbrpgs, 1138 mlx5_os_get_umem_id 1139 (txq_obj->sq_dbrec_page->umem), 1140 txq_obj->sq_dbrec_offset)); 1141 txq_obj->sq_dbrec_page = NULL; 1142 } 1143 } 1144 1145 /** 1146 * Release DevX Tx CQ resources. 1147 * 1148 * @param txq_obj 1149 * DevX Tx queue object. 1150 */ 1151 static void 1152 mlx5_txq_release_devx_cq_resources(struct mlx5_txq_obj *txq_obj) 1153 { 1154 if (txq_obj->cq_devx) 1155 claim_zero(mlx5_devx_cmd_destroy(txq_obj->cq_devx)); 1156 if (txq_obj->cq_umem) 1157 claim_zero(mlx5_os_umem_dereg(txq_obj->cq_umem)); 1158 if (txq_obj->cq_buf) 1159 mlx5_free(txq_obj->cq_buf); 1160 if (txq_obj->cq_dbrec_page) 1161 claim_zero(mlx5_release_dbr(&txq_obj->txq_ctrl->priv->dbrpgs, 1162 mlx5_os_get_umem_id 1163 (txq_obj->cq_dbrec_page->umem), 1164 txq_obj->cq_dbrec_offset)); 1165 } 1166 1167 /** 1168 * Destroy the Tx queue DevX object. 1169 * 1170 * @param txq_obj 1171 * Txq object to destroy. 1172 */ 1173 static void 1174 mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj) 1175 { 1176 mlx5_txq_release_devx_sq_resources(txq_obj); 1177 mlx5_txq_release_devx_cq_resources(txq_obj); 1178 } 1179 1180 /** 1181 * Create a DevX CQ object and its resources for an Tx queue. 1182 * 1183 * @param dev 1184 * Pointer to Ethernet device. 1185 * @param idx 1186 * Queue index in DPDK Tx queue array. 1187 * 1188 * @return 1189 * Number of CQEs in CQ, 0 otherwise and rte_errno is set. 1190 */ 1191 static uint32_t 1192 mlx5_txq_create_devx_cq_resources(struct rte_eth_dev *dev, uint16_t idx) 1193 { 1194 struct mlx5_priv *priv = dev->data->dev_private; 1195 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1196 struct mlx5_txq_ctrl *txq_ctrl = 1197 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1198 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1199 struct mlx5_devx_cq_attr cq_attr = { 0 }; 1200 struct mlx5_cqe *cqe; 1201 size_t page_size; 1202 size_t alignment; 1203 uint32_t cqe_n; 1204 uint32_t i; 1205 int ret; 1206 1207 MLX5_ASSERT(txq_data); 1208 MLX5_ASSERT(txq_obj); 1209 page_size = rte_mem_page_size(); 1210 if (page_size == (size_t)-1) { 1211 DRV_LOG(ERR, "Failed to get mem page size."); 1212 rte_errno = ENOMEM; 1213 return 0; 1214 } 1215 /* Allocate memory buffer for CQEs. */ 1216 alignment = MLX5_CQE_BUF_ALIGNMENT; 1217 if (alignment == (size_t)-1) { 1218 DRV_LOG(ERR, "Failed to get CQE buf alignment."); 1219 rte_errno = ENOMEM; 1220 return 0; 1221 } 1222 /* Create the Completion Queue. */ 1223 cqe_n = (1UL << txq_data->elts_n) / MLX5_TX_COMP_THRESH + 1224 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; 1225 cqe_n = 1UL << log2above(cqe_n); 1226 if (cqe_n > UINT16_MAX) { 1227 DRV_LOG(ERR, 1228 "Port %u Tx queue %u requests to many CQEs %u.", 1229 dev->data->port_id, txq_data->idx, cqe_n); 1230 rte_errno = EINVAL; 1231 return 0; 1232 } 1233 txq_obj->cq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 1234 cqe_n * sizeof(struct mlx5_cqe), 1235 alignment, 1236 priv->sh->numa_node); 1237 if (!txq_obj->cq_buf) { 1238 DRV_LOG(ERR, 1239 "Port %u Tx queue %u cannot allocate memory (CQ).", 1240 dev->data->port_id, txq_data->idx); 1241 rte_errno = ENOMEM; 1242 return 0; 1243 } 1244 /* Register allocated buffer in user space with DevX. */ 1245 txq_obj->cq_umem = mlx5_os_umem_reg(priv->sh->ctx, 1246 (void *)txq_obj->cq_buf, 1247 cqe_n * sizeof(struct mlx5_cqe), 1248 IBV_ACCESS_LOCAL_WRITE); 1249 if (!txq_obj->cq_umem) { 1250 rte_errno = errno; 1251 DRV_LOG(ERR, 1252 "Port %u Tx queue %u cannot register memory (CQ).", 1253 dev->data->port_id, txq_data->idx); 1254 goto error; 1255 } 1256 /* Allocate doorbell record for completion queue. */ 1257 txq_obj->cq_dbrec_offset = mlx5_get_dbr(priv->sh->ctx, 1258 &priv->dbrpgs, 1259 &txq_obj->cq_dbrec_page); 1260 if (txq_obj->cq_dbrec_offset < 0) { 1261 rte_errno = errno; 1262 DRV_LOG(ERR, "Failed to allocate CQ door-bell."); 1263 goto error; 1264 } 1265 cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? 1266 MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; 1267 cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar); 1268 cq_attr.eqn = priv->sh->eqn; 1269 cq_attr.q_umem_valid = 1; 1270 cq_attr.q_umem_offset = (uintptr_t)txq_obj->cq_buf % page_size; 1271 cq_attr.q_umem_id = mlx5_os_get_umem_id(txq_obj->cq_umem); 1272 cq_attr.db_umem_valid = 1; 1273 cq_attr.db_umem_offset = txq_obj->cq_dbrec_offset; 1274 cq_attr.db_umem_id = mlx5_os_get_umem_id(txq_obj->cq_dbrec_page->umem); 1275 cq_attr.log_cq_size = rte_log2_u32(cqe_n); 1276 cq_attr.log_page_size = rte_log2_u32(page_size); 1277 /* Create completion queue object with DevX. */ 1278 txq_obj->cq_devx = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr); 1279 if (!txq_obj->cq_devx) { 1280 rte_errno = errno; 1281 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.", 1282 dev->data->port_id, idx); 1283 goto error; 1284 } 1285 /* Initial fill CQ buffer with invalid CQE opcode. */ 1286 cqe = (struct mlx5_cqe *)txq_obj->cq_buf; 1287 for (i = 0; i < cqe_n; i++) { 1288 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK; 1289 ++cqe; 1290 } 1291 return cqe_n; 1292 error: 1293 ret = rte_errno; 1294 mlx5_txq_release_devx_cq_resources(txq_obj); 1295 rte_errno = ret; 1296 return 0; 1297 } 1298 1299 /** 1300 * Create a SQ object and its resources using DevX. 1301 * 1302 * @param dev 1303 * Pointer to Ethernet device. 1304 * @param idx 1305 * Queue index in DPDK Tx queue array. 1306 * 1307 * @return 1308 * Number of WQEs in SQ, 0 otherwise and rte_errno is set. 1309 */ 1310 static uint32_t 1311 mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx) 1312 { 1313 struct mlx5_priv *priv = dev->data->dev_private; 1314 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1315 struct mlx5_txq_ctrl *txq_ctrl = 1316 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1317 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1318 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 1319 size_t page_size; 1320 uint32_t wqe_n; 1321 int ret; 1322 1323 MLX5_ASSERT(txq_data); 1324 MLX5_ASSERT(txq_obj); 1325 page_size = rte_mem_page_size(); 1326 if (page_size == (size_t)-1) { 1327 DRV_LOG(ERR, "Failed to get mem page size."); 1328 rte_errno = ENOMEM; 1329 return 0; 1330 } 1331 wqe_n = RTE_MIN(1UL << txq_data->elts_n, 1332 (uint32_t)priv->sh->device_attr.max_qp_wr); 1333 txq_obj->sq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 1334 wqe_n * sizeof(struct mlx5_wqe), 1335 page_size, priv->sh->numa_node); 1336 if (!txq_obj->sq_buf) { 1337 DRV_LOG(ERR, 1338 "Port %u Tx queue %u cannot allocate memory (SQ).", 1339 dev->data->port_id, txq_data->idx); 1340 rte_errno = ENOMEM; 1341 goto error; 1342 } 1343 /* Register allocated buffer in user space with DevX. */ 1344 txq_obj->sq_umem = mlx5_os_umem_reg 1345 (priv->sh->ctx, 1346 (void *)txq_obj->sq_buf, 1347 wqe_n * sizeof(struct mlx5_wqe), 1348 IBV_ACCESS_LOCAL_WRITE); 1349 if (!txq_obj->sq_umem) { 1350 rte_errno = errno; 1351 DRV_LOG(ERR, 1352 "Port %u Tx queue %u cannot register memory (SQ).", 1353 dev->data->port_id, txq_data->idx); 1354 goto error; 1355 } 1356 /* Allocate doorbell record for send queue. */ 1357 txq_obj->sq_dbrec_offset = mlx5_get_dbr(priv->sh->ctx, 1358 &priv->dbrpgs, 1359 &txq_obj->sq_dbrec_page); 1360 if (txq_obj->sq_dbrec_offset < 0) { 1361 rte_errno = errno; 1362 DRV_LOG(ERR, "Failed to allocate SQ door-bell."); 1363 goto error; 1364 } 1365 sq_attr.tis_lst_sz = 1; 1366 sq_attr.tis_num = priv->sh->tis->id; 1367 sq_attr.state = MLX5_SQC_STATE_RST; 1368 sq_attr.cqn = txq_obj->cq_devx->id; 1369 sq_attr.flush_in_error_en = 1; 1370 sq_attr.allow_multi_pkt_send_wqe = !!priv->config.mps; 1371 sq_attr.allow_swp = !!priv->config.swp; 1372 sq_attr.min_wqe_inline_mode = priv->config.hca_attr.vport_inline_mode; 1373 sq_attr.wq_attr.uar_page = 1374 mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar); 1375 sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 1376 sq_attr.wq_attr.pd = priv->sh->pdn; 1377 sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 1378 sq_attr.wq_attr.log_wq_sz = log2above(wqe_n); 1379 sq_attr.wq_attr.dbr_umem_valid = 1; 1380 sq_attr.wq_attr.dbr_addr = txq_obj->sq_dbrec_offset; 1381 sq_attr.wq_attr.dbr_umem_id = 1382 mlx5_os_get_umem_id(txq_obj->sq_dbrec_page->umem); 1383 sq_attr.wq_attr.wq_umem_valid = 1; 1384 sq_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(txq_obj->sq_umem); 1385 sq_attr.wq_attr.wq_umem_offset = (uintptr_t)txq_obj->sq_buf % page_size; 1386 /* Create Send Queue object with DevX. */ 1387 txq_obj->sq_devx = mlx5_devx_cmd_create_sq(priv->sh->ctx, &sq_attr); 1388 if (!txq_obj->sq_devx) { 1389 rte_errno = errno; 1390 DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.", 1391 dev->data->port_id, idx); 1392 goto error; 1393 } 1394 return wqe_n; 1395 error: 1396 ret = rte_errno; 1397 mlx5_txq_release_devx_sq_resources(txq_obj); 1398 rte_errno = ret; 1399 return 0; 1400 } 1401 #endif 1402 1403 /** 1404 * Create the Tx queue DevX object. 1405 * 1406 * @param dev 1407 * Pointer to Ethernet device. 1408 * @param idx 1409 * Queue index in DPDK Tx queue array. 1410 * 1411 * @return 1412 * 0 on success, a negative errno value otherwise and rte_errno is set. 1413 */ 1414 int 1415 mlx5_txq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 1416 { 1417 struct mlx5_priv *priv = dev->data->dev_private; 1418 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1419 struct mlx5_txq_ctrl *txq_ctrl = 1420 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1421 1422 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) 1423 return mlx5_txq_obj_hairpin_new(dev, idx); 1424 #if !defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) && defined(HAVE_INFINIBAND_VERBS_H) 1425 DRV_LOG(ERR, "Port %u Tx queue %u cannot create with DevX, no UAR.", 1426 dev->data->port_id, idx); 1427 rte_errno = ENOMEM; 1428 return -rte_errno; 1429 #else 1430 struct mlx5_dev_ctx_shared *sh = priv->sh; 1431 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1432 void *reg_addr; 1433 uint32_t cqe_n; 1434 uint32_t wqe_n; 1435 int ret = 0; 1436 1437 MLX5_ASSERT(txq_data); 1438 MLX5_ASSERT(txq_obj); 1439 txq_obj->txq_ctrl = txq_ctrl; 1440 txq_obj->dev = dev; 1441 cqe_n = mlx5_txq_create_devx_cq_resources(dev, idx); 1442 if (!cqe_n) { 1443 rte_errno = errno; 1444 goto error; 1445 } 1446 txq_data->cqe_n = log2above(cqe_n); 1447 txq_data->cqe_s = 1 << txq_data->cqe_n; 1448 txq_data->cqe_m = txq_data->cqe_s - 1; 1449 txq_data->cqes = (volatile struct mlx5_cqe *)txq_obj->cq_buf; 1450 txq_data->cq_ci = 0; 1451 txq_data->cq_pi = 0; 1452 txq_data->cq_db = (volatile uint32_t *)(txq_obj->cq_dbrec_page->dbrs + 1453 txq_obj->cq_dbrec_offset); 1454 *txq_data->cq_db = 0; 1455 /* Create Send Queue object with DevX. */ 1456 wqe_n = mlx5_txq_create_devx_sq_resources(dev, idx); 1457 if (!wqe_n) { 1458 rte_errno = errno; 1459 goto error; 1460 } 1461 /* Create the Work Queue. */ 1462 txq_data->wqe_n = log2above(wqe_n); 1463 txq_data->wqe_s = 1 << txq_data->wqe_n; 1464 txq_data->wqe_m = txq_data->wqe_s - 1; 1465 txq_data->wqes = (struct mlx5_wqe *)txq_obj->sq_buf; 1466 txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; 1467 txq_data->wqe_ci = 0; 1468 txq_data->wqe_pi = 0; 1469 txq_data->wqe_comp = 0; 1470 txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; 1471 txq_data->qp_db = (volatile uint32_t *) 1472 (txq_obj->sq_dbrec_page->dbrs + 1473 txq_obj->sq_dbrec_offset + 1474 MLX5_SND_DBR * sizeof(uint32_t)); 1475 *txq_data->qp_db = 0; 1476 txq_data->qp_num_8s = txq_obj->sq_devx->id << 8; 1477 /* Change Send Queue state to Ready-to-Send. */ 1478 ret = mlx5_devx_modify_sq(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0); 1479 if (ret) { 1480 rte_errno = errno; 1481 DRV_LOG(ERR, 1482 "Port %u Tx queue %u SQ state to SQC_STATE_RDY failed.", 1483 dev->data->port_id, idx); 1484 goto error; 1485 } 1486 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1487 /* 1488 * If using DevX need to query and store TIS transport domain value. 1489 * This is done once per port. 1490 * Will use this value on Rx, when creating matching TIR. 1491 */ 1492 if (!priv->sh->tdn) 1493 priv->sh->tdn = priv->sh->td->id; 1494 #endif 1495 MLX5_ASSERT(sh->tx_uar); 1496 reg_addr = mlx5_os_get_devx_uar_reg_addr(sh->tx_uar); 1497 MLX5_ASSERT(reg_addr); 1498 txq_ctrl->bf_reg = reg_addr; 1499 txq_ctrl->uar_mmap_offset = 1500 mlx5_os_get_devx_uar_mmap_offset(sh->tx_uar); 1501 txq_uar_init(txq_ctrl); 1502 dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 1503 return 0; 1504 error: 1505 ret = rte_errno; /* Save rte_errno before cleanup. */ 1506 mlx5_txq_release_devx_resources(txq_obj); 1507 rte_errno = ret; /* Restore rte_errno. */ 1508 return -rte_errno; 1509 #endif 1510 } 1511 1512 /** 1513 * Release an Tx DevX queue object. 1514 * 1515 * @param txq_obj 1516 * DevX Tx queue object. 1517 */ 1518 void 1519 mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj) 1520 { 1521 MLX5_ASSERT(txq_obj); 1522 if (txq_obj->txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 1523 if (txq_obj->tis) 1524 claim_zero(mlx5_devx_cmd_destroy(txq_obj->tis)); 1525 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1526 } else { 1527 mlx5_txq_release_devx_resources(txq_obj); 1528 #endif 1529 } 1530 } 1531 1532 struct mlx5_obj_ops devx_obj_ops = { 1533 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 1534 .rxq_obj_new = mlx5_rxq_devx_obj_new, 1535 .rxq_event_get = mlx5_rx_devx_get_event, 1536 .rxq_obj_modify = mlx5_devx_modify_rq, 1537 .rxq_obj_release = mlx5_rxq_devx_obj_release, 1538 .ind_table_new = mlx5_devx_ind_table_new, 1539 .ind_table_modify = mlx5_devx_ind_table_modify, 1540 .ind_table_destroy = mlx5_devx_ind_table_destroy, 1541 .hrxq_new = mlx5_devx_hrxq_new, 1542 .hrxq_destroy = mlx5_devx_tir_destroy, 1543 .hrxq_modify = mlx5_devx_hrxq_modify, 1544 .drop_action_create = mlx5_devx_drop_action_create, 1545 .drop_action_destroy = mlx5_devx_drop_action_destroy, 1546 .txq_obj_new = mlx5_txq_devx_obj_new, 1547 .txq_obj_modify = mlx5_devx_modify_sq, 1548 .txq_obj_release = mlx5_txq_devx_obj_release, 1549 }; 1550